2024-11-07 17:17:17,397 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-07 17:17:17,415 main DEBUG Took 0.014612 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 17:17:17,417 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 17:17:17,418 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 17:17:17,450 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 17:17:17,452 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,465 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 17:17:17,481 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,483 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,484 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,484 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,485 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,485 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,486 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,488 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,489 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,491 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,492 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,493 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,494 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,495 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 17:17:17,496 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,497 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 17:17:17,499 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 17:17:17,500 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 17:17:17,503 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 17:17:17,503 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 17:17:17,505 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 17:17:17,505 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 17:17:17,516 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 17:17:17,519 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 17:17:17,521 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 17:17:17,522 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 17:17:17,522 main DEBUG createAppenders(={Console}) 2024-11-07 17:17:17,523 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-07 17:17:17,524 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-07 17:17:17,524 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-07 17:17:17,525 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 17:17:17,525 main DEBUG OutputStream closed 2024-11-07 17:17:17,525 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 17:17:17,526 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 17:17:17,526 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-07 17:17:17,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 17:17:17,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 17:17:17,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 17:17:17,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 17:17:17,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 17:17:17,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 17:17:17,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 17:17:17,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 17:17:17,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 17:17:17,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 17:17:17,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 17:17:17,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 17:17:17,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 17:17:17,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 17:17:17,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 17:17:17,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 17:17:17,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 17:17:17,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 17:17:17,641 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 17:17:17,642 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-07 17:17:17,642 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 17:17:17,643 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-07T17:17:17,955 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-07 17:17:17,959 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 17:17:17,960 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T17:17:18,063 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34 2024-11-07T17:17:18,111 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T17:17:18,115 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1, deleteOnExit=true 2024-11-07T17:17:18,116 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-07T17:17:18,117 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/test.cache.data in system properties and HBase conf 2024-11-07T17:17:18,118 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T17:17:18,118 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/hadoop.log.dir in system properties and HBase conf 2024-11-07T17:17:18,119 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T17:17:18,120 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T17:17:18,120 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-07T17:17:18,265 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T17:17:18,392 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T17:17:18,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T17:17:18,398 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T17:17:18,398 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T17:17:18,399 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T17:17:18,400 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T17:17:18,400 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T17:17:18,401 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T17:17:18,402 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T17:17:18,402 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T17:17:18,403 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/nfs.dump.dir in system properties and HBase conf 2024-11-07T17:17:18,403 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/java.io.tmpdir in system properties and HBase conf 2024-11-07T17:17:18,404 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T17:17:18,404 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T17:17:18,405 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T17:17:19,486 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T17:17:19,595 INFO [Time-limited test {}] log.Log(170): Logging initialized @3141ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T17:17:19,699 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T17:17:19,810 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T17:17:19,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T17:17:19,861 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T17:17:19,863 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T17:17:19,882 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T17:17:19,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/hadoop.log.dir/,AVAILABLE} 2024-11-07T17:17:19,891 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T17:17:20,159 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/java.io.tmpdir/jetty-localhost-39857-hadoop-hdfs-3_4_1-tests_jar-_-any-7109040073749792111/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T17:17:20,180 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:39857} 2024-11-07T17:17:20,181 INFO [Time-limited test {}] server.Server(415): Started @3728ms 2024-11-07T17:17:20,769 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T17:17:20,777 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T17:17:20,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T17:17:20,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T17:17:20,780 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T17:17:20,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@534ddf03{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/hadoop.log.dir/,AVAILABLE} 2024-11-07T17:17:20,782 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@da2227{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T17:17:20,959 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a0f56e0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/java.io.tmpdir/jetty-localhost-38353-hadoop-hdfs-3_4_1-tests_jar-_-any-16154551145404735819/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T17:17:20,960 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6dd4310{HTTP/1.1, (http/1.1)}{localhost:38353} 2024-11-07T17:17:20,961 INFO [Time-limited test {}] server.Server(415): Started @4509ms 2024-11-07T17:17:21,040 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T17:17:21,615 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1/dfs/data/data1/current/BP-1071349954-172.17.0.2-1730999839187/current, will proceed with Du for space computation calculation, 2024-11-07T17:17:21,629 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1/dfs/data/data2/current/BP-1071349954-172.17.0.2-1730999839187/current, will proceed with Du for space computation calculation, 2024-11-07T17:17:21,776 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T17:17:21,845 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x277a7e2e2563c22e with lease ID 0x3a44f3b3c63bda0d: Processing first storage report for DS-eb49c79b-afe4-4824-816a-a5ce249a4450 from datanode DatanodeRegistration(127.0.0.1:39381, datanodeUuid=5710eab7-4319-48ee-8067-f4b9cc2baa08, infoPort=46003, infoSecurePort=0, ipcPort=32815, storageInfo=lv=-57;cid=testClusterID;nsid=1034411448;c=1730999839187) 2024-11-07T17:17:21,847 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x277a7e2e2563c22e with lease ID 0x3a44f3b3c63bda0d: from storage DS-eb49c79b-afe4-4824-816a-a5ce249a4450 node DatanodeRegistration(127.0.0.1:39381, datanodeUuid=5710eab7-4319-48ee-8067-f4b9cc2baa08, infoPort=46003, infoSecurePort=0, ipcPort=32815, storageInfo=lv=-57;cid=testClusterID;nsid=1034411448;c=1730999839187), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T17:17:21,847 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x277a7e2e2563c22e with lease ID 0x3a44f3b3c63bda0d: Processing first storage report for DS-4b9e9ada-af26-406c-8a11-c86cefcd4dcf from datanode DatanodeRegistration(127.0.0.1:39381, datanodeUuid=5710eab7-4319-48ee-8067-f4b9cc2baa08, infoPort=46003, infoSecurePort=0, ipcPort=32815, storageInfo=lv=-57;cid=testClusterID;nsid=1034411448;c=1730999839187) 2024-11-07T17:17:21,848 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x277a7e2e2563c22e with lease ID 0x3a44f3b3c63bda0d: from storage DS-4b9e9ada-af26-406c-8a11-c86cefcd4dcf node DatanodeRegistration(127.0.0.1:39381, datanodeUuid=5710eab7-4319-48ee-8067-f4b9cc2baa08, infoPort=46003, infoSecurePort=0, ipcPort=32815, storageInfo=lv=-57;cid=testClusterID;nsid=1034411448;c=1730999839187), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T17:17:21,885 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34 2024-11-07T17:17:21,987 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1/zookeeper_0, clientPort=57710, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T17:17:22,005 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57710 2024-11-07T17:17:22,021 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T17:17:22,025 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T17:17:22,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741825_1001 (size=7) 2024-11-07T17:17:22,750 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 with version=8 2024-11-07T17:17:22,751 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/hbase-staging 2024-11-07T17:17:22,924 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T17:17:23,242 INFO [Time-limited test {}] client.ConnectionUtils(129): master/3a0fde618c86:0 server-side Connection retries=9 2024-11-07T17:17:23,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T17:17:23,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T17:17:23,264 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T17:17:23,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T17:17:23,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T17:17:23,429 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T17:17:23,496 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T17:17:23,506 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T17:17:23,511 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T17:17:23,544 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14784 (auto-detected) 2024-11-07T17:17:23,545 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T17:17:23,567 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36799 2024-11-07T17:17:23,576 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T17:17:23,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T17:17:23,594 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36799 connecting to ZooKeeper ensemble=127.0.0.1:57710 2024-11-07T17:17:23,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:367990x0, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T17:17:23,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36799-0x10183bcea110000 connected 2024-11-07T17:17:23,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T17:17:23,701 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T17:17:23,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T17:17:23,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36799 2024-11-07T17:17:23,713 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36799 2024-11-07T17:17:23,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36799 2024-11-07T17:17:23,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36799 2024-11-07T17:17:23,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36799 2024-11-07T17:17:23,732 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392, hbase.cluster.distributed=false 2024-11-07T17:17:23,827 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/3a0fde618c86:0 server-side Connection retries=9 2024-11-07T17:17:23,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T17:17:23,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T17:17:23,828 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T17:17:23,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T17:17:23,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T17:17:23,831 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T17:17:23,835 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T17:17:23,840 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38099 2024-11-07T17:17:23,843 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T17:17:23,850 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T17:17:23,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T17:17:23,856 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T17:17:23,861 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:38099 connecting to ZooKeeper ensemble=127.0.0.1:57710 2024-11-07T17:17:23,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380990x0, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T17:17:23,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380990x0, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T17:17:23,875 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380990x0, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T17:17:23,876 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380990x0, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T17:17:23,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38099 2024-11-07T17:17:23,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38099 2024-11-07T17:17:23,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38099 2024-11-07T17:17:23,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38099-0x10183bcea110001 connected 2024-11-07T17:17:23,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38099 2024-11-07T17:17:23,885 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38099 2024-11-07T17:17:23,892 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/3a0fde618c86,36799,1730999842914 2024-11-07T17:17:23,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T17:17:23,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T17:17:23,905 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3a0fde618c86,36799,1730999842914 2024-11-07T17:17:23,911 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3a0fde618c86:36799 2024-11-07T17:17:23,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T17:17:23,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T17:17:23,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:23,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:23,938 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T17:17:23,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T17:17:23,940 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3a0fde618c86,36799,1730999842914 from backup master directory 2024-11-07T17:17:23,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3a0fde618c86,36799,1730999842914 2024-11-07T17:17:23,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T17:17:23,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T17:17:23,951 WARN [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T17:17:23,951 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3a0fde618c86,36799,1730999842914 2024-11-07T17:17:23,954 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T17:17:23,960 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T17:17:24,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741826_1002 (size=42) 2024-11-07T17:17:24,059 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/hbase.id with ID: 7bbf0df2-befc-4458-a4b8-0d6886ec1be9 2024-11-07T17:17:24,110 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T17:17:24,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741827_1003 (size=196) 2024-11-07T17:17:24,182 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:17:24,186 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T17:17:24,205 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:17:24,210 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T17:17:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741828_1004 (size=1189) 2024-11-07T17:17:24,268 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store 2024-11-07T17:17:24,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741829_1005 (size=34) 2024-11-07T17:17:24,295 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T17:17:24,295 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:24,297 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T17:17:24,297 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:17:24,297 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:17:24,297 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T17:17:24,298 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:17:24,298 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:17:24,298 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T17:17:24,300 WARN [master/3a0fde618c86:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/.initializing 2024-11-07T17:17:24,301 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/WALs/3a0fde618c86,36799,1730999842914 2024-11-07T17:17:24,308 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T17:17:24,321 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a0fde618c86%2C36799%2C1730999842914, suffix=, logDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/WALs/3a0fde618c86,36799,1730999842914, archiveDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/oldWALs, maxLogs=10 2024-11-07T17:17:24,345 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/WALs/3a0fde618c86,36799,1730999842914/3a0fde618c86%2C36799%2C1730999842914.1730999844326, exclude list is [], retry=0 2024-11-07T17:17:24,364 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-eb49c79b-afe4-4824-816a-a5ce249a4450,DISK] 2024-11-07T17:17:24,367 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T17:17:24,427 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/WALs/3a0fde618c86,36799,1730999842914/3a0fde618c86%2C36799%2C1730999842914.1730999844326 2024-11-07T17:17:24,431 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46003:46003)] 2024-11-07T17:17:24,431 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:17:24,432 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:24,437 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,438 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T17:17:24,527 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:24,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T17:17:24,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,538 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T17:17:24,539 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:24,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:17:24,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T17:17:24,546 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:24,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:17:24,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T17:17:24,552 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:24,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:17:24,557 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,559 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,569 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T17:17:24,574 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T17:17:24,580 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:17:24,582 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70710873, jitterRate=0.05367411673069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T17:17:24,587 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T17:17:24,588 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T17:17:24,623 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbcc927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:17:24,664 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-07T17:17:24,680 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T17:17:24,680 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T17:17:24,683 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T17:17:24,685 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T17:17:24,691 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 6 msec 2024-11-07T17:17:24,691 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T17:17:24,729 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T17:17:24,743 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T17:17:24,748 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-07T17:17:24,750 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T17:17:24,752 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T17:17:24,754 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-07T17:17:24,756 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T17:17:24,760 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T17:17:24,762 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-07T17:17:24,764 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T17:17:24,768 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T17:17:24,780 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T17:17:24,782 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T17:17:24,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T17:17:24,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,788 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=3a0fde618c86,36799,1730999842914, sessionid=0x10183bcea110000, setting cluster-up flag (Was=false) 2024-11-07T17:17:24,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T17:17:24,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,810 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T17:17:24,812 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3a0fde618c86,36799,1730999842914 2024-11-07T17:17:24,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:24,824 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T17:17:24,827 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3a0fde618c86,36799,1730999842914 2024-11-07T17:17:24,910 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3a0fde618c86:38099 2024-11-07T17:17:24,912 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1008): ClusterId : 7bbf0df2-befc-4458-a4b8-0d6886ec1be9 2024-11-07T17:17:24,915 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T17:17:24,924 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T17:17:24,925 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T17:17:24,928 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T17:17:24,929 DEBUG [RS:0;3a0fde618c86:38099 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2db72c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:17:24,931 DEBUG [RS:0;3a0fde618c86:38099 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f3692c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3a0fde618c86/172.17.0.2:0 2024-11-07T17:17:24,935 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-07T17:17:24,935 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-07T17:17:24,935 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-07T17:17:24,935 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-07T17:17:24,939 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3073): reportForDuty to master=3a0fde618c86,36799,1730999842914 with isa=3a0fde618c86/172.17.0.2:38099, startcode=1730999843825 2024-11-07T17:17:24,942 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-07T17:17:24,946 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T17:17:24,953 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3a0fde618c86,36799,1730999842914 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T17:17:24,957 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3a0fde618c86:0, corePoolSize=5, maxPoolSize=5 2024-11-07T17:17:24,958 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3a0fde618c86:0, corePoolSize=5, maxPoolSize=5 2024-11-07T17:17:24,958 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3a0fde618c86:0, corePoolSize=5, maxPoolSize=5 2024-11-07T17:17:24,958 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3a0fde618c86:0, corePoolSize=5, maxPoolSize=5 2024-11-07T17:17:24,958 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3a0fde618c86:0, corePoolSize=10, maxPoolSize=10 2024-11-07T17:17:24,959 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:24,959 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3a0fde618c86:0, corePoolSize=2, maxPoolSize=2 2024-11-07T17:17:24,959 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:24,960 DEBUG [RS:0;3a0fde618c86:38099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T17:17:24,976 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T17:17:24,976 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730999874976 2024-11-07T17:17:24,977 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-07T17:17:24,978 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T17:17:24,980 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T17:17:24,983 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:24,983 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T17:17:24,985 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T17:17:24,986 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T17:17:24,986 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T17:17:24,986 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T17:17:24,991 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:24,996 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T17:17:24,998 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T17:17:24,998 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T17:17:25,014 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T17:17:25,014 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T17:17:25,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741831_1007 (size=1039) 2024-11-07T17:17:25,020 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3a0fde618c86:0:becomeActiveMaster-HFileCleaner.large.0-1730999845020,5,FailOnTimeoutGroup] 2024-11-07T17:17:25,024 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3a0fde618c86:0:becomeActiveMaster-HFileCleaner.small.0-1730999845020,5,FailOnTimeoutGroup] 2024-11-07T17:17:25,024 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,024 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T17:17:25,026 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48907, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T17:17:25,026 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,027 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,034 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36799 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 3a0fde618c86,38099,1730999843825 2024-11-07T17:17:25,037 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36799 {}] master.ServerManager(486): Registering regionserver=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:25,055 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:17:25,056 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:34871 2024-11-07T17:17:25,056 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-07T17:17:25,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T17:17:25,061 DEBUG [RS:0;3a0fde618c86:38099 {}] zookeeper.ZKUtil(111): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3a0fde618c86,38099,1730999843825 2024-11-07T17:17:25,061 WARN [RS:0;3a0fde618c86:38099 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T17:17:25,062 INFO [RS:0;3a0fde618c86:38099 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T17:17:25,062 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/WALs/3a0fde618c86,38099,1730999843825 2024-11-07T17:17:25,064 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3a0fde618c86,38099,1730999843825] 2024-11-07T17:17:25,076 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-07T17:17:25,089 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T17:17:25,103 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T17:17:25,106 INFO [RS:0;3a0fde618c86:38099 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T17:17:25,107 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,108 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-07T17:17:25,116 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,116 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,116 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,116 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,117 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,117 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,117 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3a0fde618c86:0, corePoolSize=2, maxPoolSize=2 2024-11-07T17:17:25,117 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,117 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,118 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,118 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,118 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3a0fde618c86:0, corePoolSize=1, maxPoolSize=1 2024-11-07T17:17:25,118 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3a0fde618c86:0, corePoolSize=3, maxPoolSize=3 2024-11-07T17:17:25,119 DEBUG [RS:0;3a0fde618c86:38099 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3a0fde618c86:0, corePoolSize=3, maxPoolSize=3 2024-11-07T17:17:25,120 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,120 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,120 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,120 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,121 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,38099,1730999843825-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T17:17:25,150 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T17:17:25,152 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,38099,1730999843825-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:25,181 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.Replication(204): 3a0fde618c86,38099,1730999843825 started 2024-11-07T17:17:25,181 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1767): Serving as 3a0fde618c86,38099,1730999843825, RpcServer on 3a0fde618c86/172.17.0.2:38099, sessionid=0x10183bcea110001 2024-11-07T17:17:25,182 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T17:17:25,182 DEBUG [RS:0;3a0fde618c86:38099 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3a0fde618c86,38099,1730999843825 2024-11-07T17:17:25,182 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3a0fde618c86,38099,1730999843825' 2024-11-07T17:17:25,182 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T17:17:25,183 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T17:17:25,184 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T17:17:25,185 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T17:17:25,185 DEBUG [RS:0;3a0fde618c86:38099 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3a0fde618c86,38099,1730999843825 2024-11-07T17:17:25,185 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3a0fde618c86,38099,1730999843825' 2024-11-07T17:17:25,185 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T17:17:25,186 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T17:17:25,188 DEBUG [RS:0;3a0fde618c86:38099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T17:17:25,188 INFO [RS:0;3a0fde618c86:38099 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T17:17:25,188 INFO [RS:0;3a0fde618c86:38099 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T17:17:25,420 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-07T17:17:25,421 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:17:25,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741832_1008 (size=32) 2024-11-07T17:17:25,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:25,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T17:17:25,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T17:17:25,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:25,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T17:17:25,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T17:17:25,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T17:17:25,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:25,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T17:17:25,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T17:17:25,453 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T17:17:25,453 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:25,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T17:17:25,456 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740 2024-11-07T17:17:25,457 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740 2024-11-07T17:17:25,460 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T17:17:25,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T17:17:25,468 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:17:25,469 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67036032, jitterRate=-0.0010852813720703125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T17:17:25,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T17:17:25,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T17:17:25,471 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T17:17:25,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T17:17:25,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T17:17:25,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T17:17:25,473 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T17:17:25,473 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T17:17:25,476 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T17:17:25,476 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-07T17:17:25,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T17:17:25,495 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T17:17:25,498 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T17:17:25,650 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:26,650 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:27,651 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:28,652 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:29,653 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:30,653 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:31,654 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:32,655 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T17:17:33,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T17:17:33,656 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:34,656 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:35,657 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:36,658 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:37,658 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:38,659 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:39,660 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:40,660 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:41,661 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:42,662 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:43,662 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:44,663 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:45,664 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:46,664 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:47,665 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:48,666 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:49,666 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:50,667 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:51,668 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:51,880 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:17:52,140 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:17:52,668 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:53,669 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:54,671 WARN [3a0fde618c86:36799 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T17:17:55,195 INFO [RS:0;3a0fde618c86:38099 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T17:17:55,201 INFO [RS:0;3a0fde618c86:38099 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a0fde618c86%2C38099%2C1730999843825, suffix=, logDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/WALs/3a0fde618c86,38099,1730999843825, archiveDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/oldWALs, maxLogs=32 2024-11-07T17:17:55,226 DEBUG [RS:0;3a0fde618c86:38099 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/WALs/3a0fde618c86,38099,1730999843825/3a0fde618c86%2C38099%2C1730999843825.1730999875203, exclude list is [], retry=0 2024-11-07T17:17:55,233 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-eb49c79b-afe4-4824-816a-a5ce249a4450,DISK] 2024-11-07T17:17:55,236 INFO [RS:0;3a0fde618c86:38099 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/WALs/3a0fde618c86,38099,1730999843825/3a0fde618c86%2C38099%2C1730999843825.1730999875203 2024-11-07T17:17:55,237 DEBUG [RS:0;3a0fde618c86:38099 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46003:46003)] 2024-11-07T17:17:55,425 DEBUG [3a0fde618c86:36799 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T17:17:55,430 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:55,435 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3a0fde618c86,38099,1730999843825, state=OPENING 2024-11-07T17:17:55,442 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T17:17:55,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:55,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:55,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T17:17:55,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T17:17:55,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:17:55,629 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:17:55,632 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T17:17:55,637 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37098, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T17:17:55,666 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-07T17:17:55,667 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T17:17:55,668 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T17:17:55,675 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a0fde618c86%2C38099%2C1730999843825.meta, suffix=.meta, logDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/WALs/3a0fde618c86,38099,1730999843825, archiveDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/oldWALs, maxLogs=32 2024-11-07T17:17:55,700 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/WALs/3a0fde618c86,38099,1730999843825/3a0fde618c86%2C38099%2C1730999843825.meta.1730999875678.meta, exclude list is [], retry=0 2024-11-07T17:17:55,706 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-eb49c79b-afe4-4824-816a-a5ce249a4450,DISK] 2024-11-07T17:17:55,712 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/WALs/3a0fde618c86,38099,1730999843825/3a0fde618c86%2C38099%2C1730999843825.meta.1730999875678.meta 2024-11-07T17:17:55,712 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46003:46003)] 2024-11-07T17:17:55,713 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:17:55,715 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T17:17:55,810 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T17:17:55,817 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T17:17:55,823 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T17:17:55,823 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:55,824 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-07T17:17:55,824 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-07T17:17:55,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T17:17:55,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T17:17:55,846 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:55,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T17:17:55,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T17:17:55,853 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T17:17:55,853 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:55,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T17:17:55,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T17:17:55,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T17:17:55,856 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:55,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T17:17:55,859 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740 2024-11-07T17:17:55,864 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740 2024-11-07T17:17:55,867 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T17:17:55,870 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T17:17:55,872 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63335073, jitterRate=-0.05623386800289154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T17:17:55,873 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T17:17:55,884 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730999875621 2024-11-07T17:17:55,900 DEBUG [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T17:17:55,900 INFO [RS_OPEN_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-07T17:17:55,901 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:55,904 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3a0fde618c86,38099,1730999843825, state=OPEN 2024-11-07T17:17:55,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T17:17:55,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T17:17:55,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T17:17:55,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T17:17:55,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T17:17:55,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=3a0fde618c86,38099,1730999843825 in 464 msec 2024-11-07T17:17:55,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T17:17:55,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.4350 sec 2024-11-07T17:17:55,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 31.0550 sec 2024-11-07T17:17:55,930 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730999875930, completionTime=-1 2024-11-07T17:17:55,930 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T17:17:55,930 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-07T17:17:55,972 DEBUG [hconnection-0x78d625a6-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:17:55,975 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37114, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:17:55,988 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-07T17:17:55,988 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730999935988 2024-11-07T17:17:55,988 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730999995988 2024-11-07T17:17:55,988 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 57 msec 2024-11-07T17:17:56,014 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,36799,1730999842914-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:56,015 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,36799,1730999842914-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:56,015 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,36799,1730999842914-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:56,017 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3a0fde618c86:36799, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:56,017 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:56,024 DEBUG [master/3a0fde618c86:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-07T17:17:56,026 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-07T17:17:56,028 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T17:17:56,035 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-07T17:17:56,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:17:56,040 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:56,042 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:17:56,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741835_1011 (size=358) 2024-11-07T17:17:56,066 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 306dab2b507c8881dd78df8ac96ce52f, NAME => 'hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:17:56,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741836_1012 (size=42) 2024-11-07T17:17:56,485 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:56,486 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 306dab2b507c8881dd78df8ac96ce52f, disabling compactions & flushes 2024-11-07T17:17:56,486 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:17:56,486 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:17:56,486 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. after waiting 0 ms 2024-11-07T17:17:56,486 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:17:56,486 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:17:56,486 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 306dab2b507c8881dd78df8ac96ce52f: 2024-11-07T17:17:56,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:17:56,498 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1730999876490"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999876490"}]},"ts":"1730999876490"} 2024-11-07T17:17:56,530 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:17:56,533 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:17:56,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999876533"}]},"ts":"1730999876533"} 2024-11-07T17:17:56,542 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-07T17:17:56,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=306dab2b507c8881dd78df8ac96ce52f, ASSIGN}] 2024-11-07T17:17:56,552 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=306dab2b507c8881dd78df8ac96ce52f, ASSIGN 2024-11-07T17:17:56,555 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=306dab2b507c8881dd78df8ac96ce52f, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:17:56,705 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=306dab2b507c8881dd78df8ac96ce52f, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:56,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 306dab2b507c8881dd78df8ac96ce52f, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:17:56,866 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:17:56,873 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:17:56,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 306dab2b507c8881dd78df8ac96ce52f, NAME => 'hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:17:56,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:17:56,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:56,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:17:56,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:17:56,877 INFO [StoreOpener-306dab2b507c8881dd78df8ac96ce52f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:17:56,880 INFO [StoreOpener-306dab2b507c8881dd78df8ac96ce52f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 306dab2b507c8881dd78df8ac96ce52f columnFamilyName info 2024-11-07T17:17:56,881 DEBUG [StoreOpener-306dab2b507c8881dd78df8ac96ce52f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:56,881 INFO [StoreOpener-306dab2b507c8881dd78df8ac96ce52f-1 {}] regionserver.HStore(327): Store=306dab2b507c8881dd78df8ac96ce52f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:17:56,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:17:56,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:17:56,889 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:17:56,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:17:56,894 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 306dab2b507c8881dd78df8ac96ce52f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74320728, jitterRate=0.10746514797210693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:17:56,896 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 306dab2b507c8881dd78df8ac96ce52f: 2024-11-07T17:17:56,899 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f., pid=6, masterSystemTime=1730999876865 2024-11-07T17:17:56,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:17:56,903 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:17:56,904 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=306dab2b507c8881dd78df8ac96ce52f, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:56,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T17:17:56,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 306dab2b507c8881dd78df8ac96ce52f, server=3a0fde618c86,38099,1730999843825 in 198 msec 2024-11-07T17:17:56,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T17:17:56,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=306dab2b507c8881dd78df8ac96ce52f, ASSIGN in 364 msec 2024-11-07T17:17:56,919 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:17:56,920 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999876919"}]},"ts":"1730999876919"} 2024-11-07T17:17:56,923 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-07T17:17:56,928 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:17:56,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 899 msec 2024-11-07T17:17:56,939 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-07T17:17:56,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-07T17:17:56,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:56,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:17:56,976 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-07T17:17:56,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T17:17:57,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 31 msec 2024-11-07T17:17:57,012 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-07T17:17:57,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T17:17:57,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 23 msec 2024-11-07T17:17:57,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-07T17:17:57,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-07T17:17:57,053 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 33.102sec 2024-11-07T17:17:57,055 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T17:17:57,057 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T17:17:57,058 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T17:17:57,059 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T17:17:57,059 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T17:17:57,060 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,36799,1730999842914-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T17:17:57,060 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,36799,1730999842914-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T17:17:57,070 DEBUG [master/3a0fde618c86:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-07T17:17:57,071 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T17:17:57,071 INFO [master/3a0fde618c86:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a0fde618c86,36799,1730999842914-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T17:17:57,143 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62f30d47 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a4c6ee0 2024-11-07T17:17:57,144 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-07T17:17:57,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a950e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:17:57,157 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T17:17:57,157 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T17:17:57,171 DEBUG [hconnection-0x4e627210-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:17:57,182 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37118, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:17:57,194 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=3a0fde618c86,36799,1730999842914 2024-11-07T17:17:57,215 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=3450 2024-11-07T17:17:57,221 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d6b0ab8 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6718345d 2024-11-07T17:17:57,229 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x6d6b0ab8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462fcf8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:17:57,233 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26c9f302 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d5915a 2024-11-07T17:17:57,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8ef6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:17:57,247 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T17:17:57,261 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39208, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T17:17:57,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:17:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T17:17:57,277 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:17:57,278 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:57,280 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:17:57,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-07T17:17:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T17:17:57,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741837_1013 (size=437) 2024-11-07T17:17:57,355 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 272802cb3d0cc73b8a0b7d1ab03b0b71, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:17:57,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741838_1014 (size=88) 2024-11-07T17:17:57,390 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:57,390 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 272802cb3d0cc73b8a0b7d1ab03b0b71, disabling compactions & flushes 2024-11-07T17:17:57,390 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:17:57,390 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:17:57,391 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. after waiting 0 ms 2024-11-07T17:17:57,391 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:17:57,391 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:17:57,391 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 272802cb3d0cc73b8a0b7d1ab03b0b71: 2024-11-07T17:17:57,394 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:17:57,394 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1730999877394"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999877394"}]},"ts":"1730999877394"} 2024-11-07T17:17:57,398 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:17:57,400 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:17:57,400 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999877400"}]},"ts":"1730999877400"} 2024-11-07T17:17:57,404 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:17:57,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=272802cb3d0cc73b8a0b7d1ab03b0b71, ASSIGN}] 2024-11-07T17:17:57,412 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=272802cb3d0cc73b8a0b7d1ab03b0b71, ASSIGN 2024-11-07T17:17:57,414 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=272802cb3d0cc73b8a0b7d1ab03b0b71, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:17:57,565 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=272802cb3d0cc73b8a0b7d1ab03b0b71, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:57,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 272802cb3d0cc73b8a0b7d1ab03b0b71, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:17:57,724 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:17:57,731 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:17:57,732 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 272802cb3d0cc73b8a0b7d1ab03b0b71, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:17:57,733 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false 272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:17:57,733 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:57,733 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:17:57,733 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:17:57,736 INFO [StoreOpener-272802cb3d0cc73b8a0b7d1ab03b0b71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:17:57,739 INFO [StoreOpener-272802cb3d0cc73b8a0b7d1ab03b0b71-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 272802cb3d0cc73b8a0b7d1ab03b0b71 columnFamilyName testFamily 2024-11-07T17:17:57,740 DEBUG [StoreOpener-272802cb3d0cc73b8a0b7d1ab03b0b71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:57,741 INFO [StoreOpener-272802cb3d0cc73b8a0b7d1ab03b0b71-1 {}] regionserver.HStore(327): Store=272802cb3d0cc73b8a0b7d1ab03b0b71/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:17:57,744 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:17:57,745 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:17:57,750 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:17:57,755 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:17:57,756 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 272802cb3d0cc73b8a0b7d1ab03b0b71; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61171823, jitterRate=-0.08846880495548248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:17:57,757 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 272802cb3d0cc73b8a0b7d1ab03b0b71: 2024-11-07T17:17:57,759 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71., pid=11, masterSystemTime=1730999877723 2024-11-07T17:17:57,764 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:17:57,764 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:17:57,765 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=272802cb3d0cc73b8a0b7d1ab03b0b71, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:57,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-07T17:17:57,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 272802cb3d0cc73b8a0b7d1ab03b0b71, server=3a0fde618c86,38099,1730999843825 in 200 msec 2024-11-07T17:17:57,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T17:17:57,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=272802cb3d0cc73b8a0b7d1ab03b0b71, ASSIGN in 364 msec 2024-11-07T17:17:57,778 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:17:57,778 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999877778"}]},"ts":"1730999877778"} 2024-11-07T17:17:57,781 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:17:57,785 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:17:57,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 513 msec 2024-11-07T17:17:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T17:17:58,333 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-07T17:17:58,334 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:17:58,335 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:17:58,341 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T17:17:58,342 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:17:58,342 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-07T17:17:58,345 DEBUG [hconnection-0x7e9b37d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:17:58,348 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37126, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:17:58,364 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:17:58,373 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T17:17:58,389 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:17:58,392 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x6d6b0ab8 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:17:58,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:17:58,399 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37142, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:17:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:17:58,414 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T17:17:58,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938251 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:17:58,417 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T17:17:58,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938251 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:17:58,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613365 2024-11-07T17:17:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37142 deadline: 1730999881417, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:17:58,448 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938251 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:17:58,450 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T17:17:58,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938252 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:17:58,452 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T17:17:58,453 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:17:58,454 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:17:58,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938253 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:17:58,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613363 2024-11-07T17:17:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37142 deadline: 1730999881454, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:17:58,457 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938253 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:17:58,459 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:17:58,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938254 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:17:58,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5792281934352613362 2024-11-07T17:17:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37142 deadline: 1730999881459, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:17:58,463 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:17:58,462 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938254 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:17:58,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938255 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:17:58,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5792281934352613361 2024-11-07T17:17:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37142 deadline: 1730999881463, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:17:58,467 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938255 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:17:58,469 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:17:58,470 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x26c9f302 to 127.0.0.1:57710 2024-11-07T17:17:58,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:17:58,471 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:17:58,471 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:17:58,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:17:58,478 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d6b0ab8 to 127.0.0.1:57710 2024-11-07T17:17:58,491 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=232 (was 219) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57710@0x6d6b0ab8 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$731/0x00007f196cb42220.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57710@0x6d6b0ab8-SendThread(127.0.0.1:57710) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-119066058_22 at /127.0.0.1:43394 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57710@0x26c9f302-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57710@0x26c9f302-SendThread(127.0.0.1:57710) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57710@0x26c9f302 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$731/0x00007f196cb42220.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x4e627210-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57710@0x6d6b0ab8-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=454 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 447), ProcessCount=11 (was 11), AvailableMemoryMB=3410 (was 3450) 2024-11-07T17:17:58,503 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=232, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=3410 2024-11-07T17:17:58,506 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50836dd5 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@49fd3744 2024-11-07T17:17:58,511 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x50836dd5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cde2aa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:17:58,513 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2507c6b2 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12ca9bde 2024-11-07T17:17:58,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ecb197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:17:58,523 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:17:58,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T17:17:58,527 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:17:58,528 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:58,528 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-07T17:17:58,530 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:17:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T17:17:58,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741839_1015 (size=423) 2024-11-07T17:17:58,946 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ff370301e57fc94e04c045b6033bb3cb, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:17:58,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741840_1016 (size=74) 2024-11-07T17:17:59,358 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:59,358 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ff370301e57fc94e04c045b6033bb3cb, disabling compactions & flushes 2024-11-07T17:17:59,359 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:17:59,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:17:59,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. after waiting 0 ms 2024-11-07T17:17:59,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:17:59,359 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:17:59,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ff370301e57fc94e04c045b6033bb3cb: 2024-11-07T17:17:59,361 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:17:59,361 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1730999879361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999879361"}]},"ts":"1730999879361"} 2024-11-07T17:17:59,365 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:17:59,367 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:17:59,367 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999879367"}]},"ts":"1730999879367"} 2024-11-07T17:17:59,370 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:17:59,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ff370301e57fc94e04c045b6033bb3cb, ASSIGN}] 2024-11-07T17:17:59,377 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ff370301e57fc94e04c045b6033bb3cb, ASSIGN 2024-11-07T17:17:59,378 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ff370301e57fc94e04c045b6033bb3cb, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:17:59,529 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=ff370301e57fc94e04c045b6033bb3cb, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:59,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure ff370301e57fc94e04c045b6033bb3cb, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:17:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T17:17:59,688 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:17:59,696 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:17:59,696 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => ff370301e57fc94e04c045b6033bb3cb, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:17:59,696 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:17:59,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:17:59,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:17:59,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:17:59,699 INFO [StoreOpener-ff370301e57fc94e04c045b6033bb3cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:17:59,702 INFO [StoreOpener-ff370301e57fc94e04c045b6033bb3cb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff370301e57fc94e04c045b6033bb3cb columnFamilyName testFamily 2024-11-07T17:17:59,702 DEBUG [StoreOpener-ff370301e57fc94e04c045b6033bb3cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:17:59,703 INFO [StoreOpener-ff370301e57fc94e04c045b6033bb3cb-1 {}] regionserver.HStore(327): Store=ff370301e57fc94e04c045b6033bb3cb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:17:59,705 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:17:59,705 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:17:59,709 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:17:59,713 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:17:59,714 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened ff370301e57fc94e04c045b6033bb3cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60560646, jitterRate=-0.09757605195045471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:17:59,715 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for ff370301e57fc94e04c045b6033bb3cb: 2024-11-07T17:17:59,717 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb., pid=14, masterSystemTime=1730999879688 2024-11-07T17:17:59,720 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:17:59,721 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:17:59,721 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=ff370301e57fc94e04c045b6033bb3cb, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:17:59,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T17:17:59,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure ff370301e57fc94e04c045b6033bb3cb, server=3a0fde618c86,38099,1730999843825 in 191 msec 2024-11-07T17:17:59,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-07T17:17:59,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=ff370301e57fc94e04c045b6033bb3cb, ASSIGN in 352 msec 2024-11-07T17:17:59,733 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:17:59,733 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999879733"}]},"ts":"1730999879733"} 2024-11-07T17:17:59,736 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:17:59,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:17:59,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 1.2180 sec 2024-11-07T17:17:59,934 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T17:17:59,936 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48426, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T17:18:01,364 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:18:01,396 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-07T17:18:01,398 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T17:18:01,399 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-07T17:18:01,400 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-07T17:18:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T17:18:01,546 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-07T17:18:01,547 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:01,547 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:01,552 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:01,552 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:01,553 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T17:18:01,555 DEBUG [hconnection-0x18ac8fdd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:01,558 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35638, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:01,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:01,576 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:18:01,578 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:18:03,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T17:18:03,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:03,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T17:18:03,494 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:03,495 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-07T17:18:03,495 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:03,496 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T17:18:03,496 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T17:18:03,496 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T17:18:03,496 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:08,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T17:18:08,594 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:18:10,582 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11008: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11008: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:18:10,582 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T17:18:10.582Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11008: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11008: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1988ms, rpcTimeout=1984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:18:10,583 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-07T17:18:10,584 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2507c6b2 to 127.0.0.1:57710 2024-11-07T17:18:10,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:10,584 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:10,584 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:10,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:10,584 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50836dd5 to 127.0.0.1:57710 2024-11-07T17:18:10,593 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=222 (was 232), OpenFileDescriptor=448 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=441 (was 447), ProcessCount=11 (was 11), AvailableMemoryMB=3403 (was 3410) 2024-11-07T17:18:10,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=441, ProcessCount=11, AvailableMemoryMB=3403 2024-11-07T17:18:10,603 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62e923d6 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@310cffc5 2024-11-07T17:18:10,606 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x62e923d6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12484fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:10,608 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01f4951f to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55ff6a5 2024-11-07T17:18:10,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e03924b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:10,614 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:10,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T17:18:10,617 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:10,618 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:10,618 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-07T17:18:10,619 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T17:18:10,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741841_1017 (size=421) 2024-11-07T17:18:11,031 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f859399bd9a2dff5654f21dedb4a5651, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:11,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741842_1018 (size=72) 2024-11-07T17:18:11,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:11,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing f859399bd9a2dff5654f21dedb4a5651, disabling compactions & flushes 2024-11-07T17:18:11,441 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:18:11,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:18:11,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. after waiting 0 ms 2024-11-07T17:18:11,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:18:11,441 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:18:11,441 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for f859399bd9a2dff5654f21dedb4a5651: 2024-11-07T17:18:11,443 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:11,444 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730999891443"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999891443"}]},"ts":"1730999891443"} 2024-11-07T17:18:11,446 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:11,448 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:11,448 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999891448"}]},"ts":"1730999891448"} 2024-11-07T17:18:11,450 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:11,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f859399bd9a2dff5654f21dedb4a5651, ASSIGN}] 2024-11-07T17:18:11,456 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f859399bd9a2dff5654f21dedb4a5651, ASSIGN 2024-11-07T17:18:11,457 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f859399bd9a2dff5654f21dedb4a5651, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:11,608 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=f859399bd9a2dff5654f21dedb4a5651, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:11,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure f859399bd9a2dff5654f21dedb4a5651, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T17:18:11,764 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:11,771 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:18:11,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => f859399bd9a2dff5654f21dedb4a5651, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:11,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:18:11,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:11,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:18:11,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:18:11,773 INFO [StoreOpener-f859399bd9a2dff5654f21dedb4a5651-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:18:11,776 INFO [StoreOpener-f859399bd9a2dff5654f21dedb4a5651-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f859399bd9a2dff5654f21dedb4a5651 columnFamilyName testFamily 2024-11-07T17:18:11,776 DEBUG [StoreOpener-f859399bd9a2dff5654f21dedb4a5651-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:11,776 INFO [StoreOpener-f859399bd9a2dff5654f21dedb4a5651-1 {}] regionserver.HStore(327): Store=f859399bd9a2dff5654f21dedb4a5651/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:11,777 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:18:11,778 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:18:11,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:18:11,784 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:11,785 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened f859399bd9a2dff5654f21dedb4a5651; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72164379, jitterRate=0.07533304393291473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:11,785 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for f859399bd9a2dff5654f21dedb4a5651: 2024-11-07T17:18:11,787 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651., pid=17, masterSystemTime=1730999891764 2024-11-07T17:18:11,789 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:18:11,790 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:18:11,790 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=f859399bd9a2dff5654f21dedb4a5651, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:11,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-07T17:18:11,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure f859399bd9a2dff5654f21dedb4a5651, server=3a0fde618c86,38099,1730999843825 in 182 msec 2024-11-07T17:18:11,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T17:18:11,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=f859399bd9a2dff5654f21dedb4a5651, ASSIGN in 341 msec 2024-11-07T17:18:11,799 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:11,799 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999891799"}]},"ts":"1730999891799"} 2024-11-07T17:18:11,801 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:11,805 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:11,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 1.1910 sec 2024-11-07T17:18:13,403 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-07T17:18:13,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T17:18:13,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T17:18:13,625 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-07T17:18:13,625 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:13,625 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:13,631 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:13,631 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:13,631 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T17:18:13,634 DEBUG [hconnection-0x15edec57-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:13,636 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42210, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:13,650 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:13,656 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:18:13,657 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:18:13,658 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x62e923d6 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:18:13,659 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x62e923d6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:13,661 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42212, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:13,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:18:13,666 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:18:13,666 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938267 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:13,669 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:18:13,669 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938267 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:13,670 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T17:18:15,094 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:18:17,305 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:18:21,880 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:18:22,673 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9003 2024-11-07T17:18:22,673 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01f4951f to 127.0.0.1:57710 2024-11-07T17:18:22,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:22,673 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:22,673 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938267 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:18:22,673 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:22,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:22,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613349' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T17:18:22,673 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62e923d6 to 127.0.0.1:57710 2024-11-07T17:18:22,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:42212 deadline: 1730999908673, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613349' does not exist 2024-11-07T17:18:22,677 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:18:22,674 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a0fde618c86,38099,1730999843825 for closing scanner id = -5792281934352613349 for f859399bd9a2dff5654f21dedb4a5651 of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T17:18:22,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=218 (was 222), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 441), ProcessCount=11 (was 11), AvailableMemoryMB=4230 (was 3403) - AvailableMemoryMB LEAK? - 2024-11-07T17:18:22,693 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=218, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=4230 2024-11-07T17:18:22,695 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4960dc7a to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f569f35 2024-11-07T17:18:22,698 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x4960dc7a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66934c8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:22,699 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09f703be to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d866882 2024-11-07T17:18:22,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224afa44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:22,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T17:18:22,709 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:22,709 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:22,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-07T17:18:22,710 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T17:18:22,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741843_1019 (size=432) 2024-11-07T17:18:23,123 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a64612bdb26c784d77aca9c346e0664d, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:23,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741844_1020 (size=83) 2024-11-07T17:18:23,170 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:18:23,533 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:23,533 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing a64612bdb26c784d77aca9c346e0664d, disabling compactions & flushes 2024-11-07T17:18:23,533 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:18:23,533 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:18:23,533 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. after waiting 0 ms 2024-11-07T17:18:23,533 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:18:23,533 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:18:23,533 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for a64612bdb26c784d77aca9c346e0664d: 2024-11-07T17:18:23,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:23,536 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1730999903535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999903535"}]},"ts":"1730999903535"} 2024-11-07T17:18:23,538 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:23,540 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:23,540 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999903540"}]},"ts":"1730999903540"} 2024-11-07T17:18:23,542 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:23,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a64612bdb26c784d77aca9c346e0664d, ASSIGN}] 2024-11-07T17:18:23,552 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a64612bdb26c784d77aca9c346e0664d, ASSIGN 2024-11-07T17:18:23,553 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a64612bdb26c784d77aca9c346e0664d, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:23,703 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=a64612bdb26c784d77aca9c346e0664d, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:23,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure a64612bdb26c784d77aca9c346e0664d, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T17:18:23,859 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:23,866 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:18:23,866 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => a64612bdb26c784d77aca9c346e0664d, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:23,867 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:18:23,867 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:23,867 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:18:23,867 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:18:23,869 INFO [StoreOpener-a64612bdb26c784d77aca9c346e0664d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:18:23,872 INFO [StoreOpener-a64612bdb26c784d77aca9c346e0664d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64612bdb26c784d77aca9c346e0664d columnFamilyName testFamily 2024-11-07T17:18:23,872 DEBUG [StoreOpener-a64612bdb26c784d77aca9c346e0664d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:23,873 INFO [StoreOpener-a64612bdb26c784d77aca9c346e0664d-1 {}] regionserver.HStore(327): Store=a64612bdb26c784d77aca9c346e0664d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:23,874 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:18:23,874 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:18:23,877 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:18:23,883 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:23,884 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened a64612bdb26c784d77aca9c346e0664d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64119150, jitterRate=-0.04455021023750305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:23,884 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for a64612bdb26c784d77aca9c346e0664d: 2024-11-07T17:18:23,885 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d., pid=20, masterSystemTime=1730999903859 2024-11-07T17:18:23,888 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:18:23,889 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:18:23,890 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=a64612bdb26c784d77aca9c346e0664d, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:23,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T17:18:23,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure a64612bdb26c784d77aca9c346e0664d, server=3a0fde618c86,38099,1730999843825 in 186 msec 2024-11-07T17:18:23,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-07T17:18:23,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=a64612bdb26c784d77aca9c346e0664d, ASSIGN in 345 msec 2024-11-07T17:18:23,899 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:23,900 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999903900"}]},"ts":"1730999903900"} 2024-11-07T17:18:23,902 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:23,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:23,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 1.2010 sec 2024-11-07T17:18:25,232 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:34871,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:18:25,405 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-07T17:18:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T17:18:25,715 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-07T17:18:25,716 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:25,716 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:25,720 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:25,720 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:25,720 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-07T17:18:25,724 DEBUG [hconnection-0x69f0ffa1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:25,726 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34264, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:25,739 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:25,742 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T17:18:25,744 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T17:18:25,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:25,746 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T17:18:25,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938271 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:25,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613345 2024-11-07T17:18:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:34264 deadline: 1730999906747, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:18:25,749 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:18:25,751 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T17:18:25,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938272 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:25,752 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T17:18:25,752 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:18:25,754 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:18:25,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938273 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:25,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613343 2024-11-07T17:18:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:34264 deadline: 1730999906754, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:18:25,756 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:18:25,757 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:18:25,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938274 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:25,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5792281934352613342 2024-11-07T17:18:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:34264 deadline: 1730999906758, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:18:25,759 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:18:25,761 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:18:25,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938275 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:25,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5792281934352613341 2024-11-07T17:18:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:34264 deadline: 1730999906761, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:18:25,762 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:18:25,764 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:18:25,765 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x09f703be to 127.0.0.1:57710 2024-11-07T17:18:25,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:25,765 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:25,765 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:25,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:25,766 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4960dc7a to 127.0.0.1:57710 2024-11-07T17:18:25,778 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=224 (was 218) - Thread LEAK? -, OpenFileDescriptor=454 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=410 (was 403) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4106 (was 4230) 2024-11-07T17:18:25,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=224, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=410, ProcessCount=11, AvailableMemoryMB=4105 2024-11-07T17:18:25,789 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x30e0b28a to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@952c087 2024-11-07T17:18:25,793 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x30e0b28a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f63a16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:25,794 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1766338c to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5938dd66 2024-11-07T17:18:25,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c794029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:25,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T17:18:25,804 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:25,805 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:25,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-07T17:18:25,806 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T17:18:25,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741845_1021 (size=426) 2024-11-07T17:18:26,054 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:18:26,218 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => aa009bbf2796b670b2b6eb293cb23b7d, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:26,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741846_1022 (size=77) 2024-11-07T17:18:26,628 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:26,628 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing aa009bbf2796b670b2b6eb293cb23b7d, disabling compactions & flushes 2024-11-07T17:18:26,628 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:18:26,628 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:18:26,628 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. after waiting 0 ms 2024-11-07T17:18:26,628 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:18:26,628 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:18:26,629 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for aa009bbf2796b670b2b6eb293cb23b7d: 2024-11-07T17:18:26,631 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:26,631 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1730999906631"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999906631"}]},"ts":"1730999906631"} 2024-11-07T17:18:26,633 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:26,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:26,635 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999906635"}]},"ts":"1730999906635"} 2024-11-07T17:18:26,637 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:26,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=aa009bbf2796b670b2b6eb293cb23b7d, ASSIGN}] 2024-11-07T17:18:26,644 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=aa009bbf2796b670b2b6eb293cb23b7d, ASSIGN 2024-11-07T17:18:26,645 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=aa009bbf2796b670b2b6eb293cb23b7d, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:26,796 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=aa009bbf2796b670b2b6eb293cb23b7d, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:26,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure aa009bbf2796b670b2b6eb293cb23b7d, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T17:18:26,952 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:26,958 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:18:26,958 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => aa009bbf2796b670b2b6eb293cb23b7d, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:26,959 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:18:26,959 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:26,959 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:18:26,960 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:18:26,962 INFO [StoreOpener-aa009bbf2796b670b2b6eb293cb23b7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:18:26,964 INFO [StoreOpener-aa009bbf2796b670b2b6eb293cb23b7d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa009bbf2796b670b2b6eb293cb23b7d columnFamilyName testFamily 2024-11-07T17:18:26,965 DEBUG [StoreOpener-aa009bbf2796b670b2b6eb293cb23b7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:26,965 INFO [StoreOpener-aa009bbf2796b670b2b6eb293cb23b7d-1 {}] regionserver.HStore(327): Store=aa009bbf2796b670b2b6eb293cb23b7d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:26,967 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:18:26,967 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:18:26,970 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:18:26,973 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:26,973 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened aa009bbf2796b670b2b6eb293cb23b7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59749248, jitterRate=-0.10966682434082031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:26,974 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for aa009bbf2796b670b2b6eb293cb23b7d: 2024-11-07T17:18:26,975 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d., pid=23, masterSystemTime=1730999906952 2024-11-07T17:18:26,978 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:18:26,978 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:18:26,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=aa009bbf2796b670b2b6eb293cb23b7d, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:26,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-07T17:18:26,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure aa009bbf2796b670b2b6eb293cb23b7d, server=3a0fde618c86,38099,1730999843825 in 182 msec 2024-11-07T17:18:26,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T17:18:26,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=aa009bbf2796b670b2b6eb293cb23b7d, ASSIGN in 342 msec 2024-11-07T17:18:26,989 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:26,989 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999906989"}]},"ts":"1730999906989"} 2024-11-07T17:18:26,991 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:26,994 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:26,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 1.1930 sec 2024-11-07T17:18:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T17:18:28,811 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-07T17:18:28,811 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:28,811 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:28,816 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:28,816 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:28,817 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T17:18:28,819 DEBUG [hconnection-0x6cfe1923-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:28,821 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34280, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:28,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:28,835 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:18:28,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:18:28,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T17:18:29,385 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:18:30,342 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T17:18:30,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:18:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:34280 deadline: 1730999911842, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:18:30,854 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:18:30,854 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938280 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:18:30,855 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2019 2024-11-07T17:18:30,855 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1766338c to 127.0.0.1:57710 2024-11-07T17:18:30,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:30,856 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:30,856 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:30,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:30,856 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x30e0b28a to 127.0.0.1:57710 2024-11-07T17:18:30,865 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=225 (was 224) - Thread LEAK? -, OpenFileDescriptor=454 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 410), ProcessCount=11 (was 11), AvailableMemoryMB=4088 (was 4105) 2024-11-07T17:18:30,873 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=4088 2024-11-07T17:18:30,875 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59af6016 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b10e438 2024-11-07T17:18:30,879 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x59af6016 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3f6cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:30,880 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x49007d9b to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5cb8286d 2024-11-07T17:18:30,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ce1d85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:30,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T17:18:30,892 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:30,892 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:30,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-07T17:18:30,893 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T17:18:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741847_1023 (size=427) 2024-11-07T17:18:31,313 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3cfc81fcf468d7f0dd7b1f50158dd4fa, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:31,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741848_1024 (size=78) 2024-11-07T17:18:31,406 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-07T17:18:31,722 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:31,722 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 3cfc81fcf468d7f0dd7b1f50158dd4fa, disabling compactions & flushes 2024-11-07T17:18:31,722 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:18:31,722 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:18:31,722 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. after waiting 0 ms 2024-11-07T17:18:31,722 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:18:31,722 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:18:31,722 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3cfc81fcf468d7f0dd7b1f50158dd4fa: 2024-11-07T17:18:31,724 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:31,724 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1730999911724"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999911724"}]},"ts":"1730999911724"} 2024-11-07T17:18:31,727 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:31,728 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:31,728 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999911728"}]},"ts":"1730999911728"} 2024-11-07T17:18:31,730 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:31,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=3cfc81fcf468d7f0dd7b1f50158dd4fa, ASSIGN}] 2024-11-07T17:18:31,735 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=3cfc81fcf468d7f0dd7b1f50158dd4fa, ASSIGN 2024-11-07T17:18:31,736 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=3cfc81fcf468d7f0dd7b1f50158dd4fa, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:31,887 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=3cfc81fcf468d7f0dd7b1f50158dd4fa, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:31,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 3cfc81fcf468d7f0dd7b1f50158dd4fa, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T17:18:32,043 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:32,048 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:18:32,048 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => 3cfc81fcf468d7f0dd7b1f50158dd4fa, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:32,049 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:18:32,049 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:32,049 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for 3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:18:32,049 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for 3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:18:32,051 INFO [StoreOpener-3cfc81fcf468d7f0dd7b1f50158dd4fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:18:32,053 INFO [StoreOpener-3cfc81fcf468d7f0dd7b1f50158dd4fa-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3cfc81fcf468d7f0dd7b1f50158dd4fa columnFamilyName testFamily 2024-11-07T17:18:32,053 DEBUG [StoreOpener-3cfc81fcf468d7f0dd7b1f50158dd4fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:32,054 INFO [StoreOpener-3cfc81fcf468d7f0dd7b1f50158dd4fa-1 {}] regionserver.HStore(327): Store=3cfc81fcf468d7f0dd7b1f50158dd4fa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:32,055 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:18:32,055 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:18:32,058 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for 3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:18:32,060 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:32,061 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened 3cfc81fcf468d7f0dd7b1f50158dd4fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59493123, jitterRate=-0.11348338425159454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:32,063 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for 3cfc81fcf468d7f0dd7b1f50158dd4fa: 2024-11-07T17:18:32,064 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa., pid=26, masterSystemTime=1730999912042 2024-11-07T17:18:32,066 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:18:32,066 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:18:32,067 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=3cfc81fcf468d7f0dd7b1f50158dd4fa, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:32,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T17:18:32,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 3cfc81fcf468d7f0dd7b1f50158dd4fa, server=3a0fde618c86,38099,1730999843825 in 180 msec 2024-11-07T17:18:32,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-07T17:18:32,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=3cfc81fcf468d7f0dd7b1f50158dd4fa, ASSIGN in 339 msec 2024-11-07T17:18:32,076 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:32,076 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999912076"}]},"ts":"1730999912076"} 2024-11-07T17:18:32,078 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:32,082 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:32,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 1.1930 sec 2024-11-07T17:18:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T17:18:33,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:33,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T17:18:33,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:33,494 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T17:18:33,494 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T17:18:33,898 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-07T17:18:33,898 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:33,898 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:33,902 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:33,903 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:33,903 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T17:18:33,905 DEBUG [hconnection-0x69ad79a5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:33,907 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52186, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:33,917 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:33,920 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:18:33,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:18:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T17:18:35,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:18:36,922 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:18:36,923 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T17:18:36.922Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=995ms, rpcTimeout=993ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:18:36,923 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3003 2024-11-07T17:18:36,924 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x49007d9b to 127.0.0.1:57710 2024-11-07T17:18:36,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:36,924 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:36,924 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:36,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:36,924 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59af6016 to 127.0.0.1:57710 2024-11-07T17:18:36,933 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225 (was 225), OpenFileDescriptor=452 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=362 (was 393), ProcessCount=11 (was 11), AvailableMemoryMB=4028 (was 4088) 2024-11-07T17:18:36,941 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=362, ProcessCount=11, AvailableMemoryMB=4028 2024-11-07T17:18:36,943 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x740e3322 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@20a71796 2024-11-07T17:18:36,947 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x740e3322 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f51a389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:36,948 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x543cf504 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f0d1bbf 2024-11-07T17:18:36,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1fa25f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:36,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T17:18:36,957 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:36,957 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:36,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-07T17:18:36,958 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T17:18:36,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741849_1025 (size=425) 2024-11-07T17:18:36,969 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d19a44f348557665f53debac39ada87f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:36,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741850_1026 (size=76) 2024-11-07T17:18:37,378 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:37,378 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing d19a44f348557665f53debac39ada87f, disabling compactions & flushes 2024-11-07T17:18:37,378 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:18:37,378 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:18:37,378 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. after waiting 0 ms 2024-11-07T17:18:37,378 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:18:37,378 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:18:37,378 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for d19a44f348557665f53debac39ada87f: 2024-11-07T17:18:37,380 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:37,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1730999917380"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999917380"}]},"ts":"1730999917380"} 2024-11-07T17:18:37,382 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:37,383 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:37,384 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999917384"}]},"ts":"1730999917384"} 2024-11-07T17:18:37,385 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:37,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d19a44f348557665f53debac39ada87f, ASSIGN}] 2024-11-07T17:18:37,391 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d19a44f348557665f53debac39ada87f, ASSIGN 2024-11-07T17:18:37,392 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d19a44f348557665f53debac39ada87f, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:37,428 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:18:37,542 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=d19a44f348557665f53debac39ada87f, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:37,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure d19a44f348557665f53debac39ada87f, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:37,551 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:18:37,588 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-07T17:18:37,698 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:37,703 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:18:37,704 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => d19a44f348557665f53debac39ada87f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:37,704 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false d19a44f348557665f53debac39ada87f 2024-11-07T17:18:37,704 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:37,704 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for d19a44f348557665f53debac39ada87f 2024-11-07T17:18:37,705 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for d19a44f348557665f53debac39ada87f 2024-11-07T17:18:37,706 INFO [StoreOpener-d19a44f348557665f53debac39ada87f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d19a44f348557665f53debac39ada87f 2024-11-07T17:18:37,708 INFO [StoreOpener-d19a44f348557665f53debac39ada87f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d19a44f348557665f53debac39ada87f columnFamilyName testFamily 2024-11-07T17:18:37,709 DEBUG [StoreOpener-d19a44f348557665f53debac39ada87f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:37,709 INFO [StoreOpener-d19a44f348557665f53debac39ada87f-1 {}] regionserver.HStore(327): Store=d19a44f348557665f53debac39ada87f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:37,710 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f 2024-11-07T17:18:37,711 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f 2024-11-07T17:18:37,714 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for d19a44f348557665f53debac39ada87f 2024-11-07T17:18:37,717 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:37,717 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened d19a44f348557665f53debac39ada87f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60212900, jitterRate=-0.10275787115097046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:37,718 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for d19a44f348557665f53debac39ada87f: 2024-11-07T17:18:37,719 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f., pid=29, masterSystemTime=1730999917698 2024-11-07T17:18:37,722 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:18:37,722 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:18:37,722 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=d19a44f348557665f53debac39ada87f, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:37,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T17:18:37,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure d19a44f348557665f53debac39ada87f, server=3a0fde618c86,38099,1730999843825 in 180 msec 2024-11-07T17:18:37,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T17:18:37,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d19a44f348557665f53debac39ada87f, ASSIGN in 339 msec 2024-11-07T17:18:37,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:37,732 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999917732"}]},"ts":"1730999917732"} 2024-11-07T17:18:37,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:37,737 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:37,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 783 msec 2024-11-07T17:18:37,817 INFO [master/3a0fde618c86:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T17:18:37,817 INFO [master/3a0fde618c86:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T17:18:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T17:18:37,961 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-07T17:18:37,961 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:37,961 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:37,966 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:37,966 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:37,966 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T17:18:37,968 DEBUG [hconnection-0x5176280c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:37,971 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52196, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:37,981 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:37,984 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:18:37,985 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:18:37,986 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x740e3322 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:18:37,987 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x740e3322 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:37,989 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52204, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:18:37,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:18:37,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938292 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:37,995 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:18:37,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938292 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:37,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T17:18:41,003 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3007 2024-11-07T17:18:41,003 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x543cf504 to 127.0.0.1:57710 2024-11-07T17:18:41,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:41,003 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:41,003 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938292 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:18:41,003 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:41,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:41,003 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x740e3322 to 127.0.0.1:57710 2024-11-07T17:18:41,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613324' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T17:18:41,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:52204 deadline: 1730999922003, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613324' does not exist 2024-11-07T17:18:41,006 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:18:41,004 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a0fde618c86,38099,1730999843825 for closing scanner id = -5792281934352613324 for d19a44f348557665f53debac39ada87f of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T17:18:41,014 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=228 (was 225) - Thread LEAK? -, OpenFileDescriptor=454 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 362), ProcessCount=11 (was 11), AvailableMemoryMB=3954 (was 4028) 2024-11-07T17:18:41,023 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=3953 2024-11-07T17:18:41,026 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4768f27f to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e759927 2024-11-07T17:18:41,029 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x4768f27f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2235ec52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:41,031 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d2a3f0e to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5fccde62 2024-11-07T17:18:41,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3734ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:41,037 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:41,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-07T17:18:41,040 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:41,040 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:41,040 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-07T17:18:41,041 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T17:18:41,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741851_1027 (size=417) 2024-11-07T17:18:41,453 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3ebb330a2ef5c6ab0e856e72db2c5e93, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:41,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741852_1028 (size=68) 2024-11-07T17:18:41,467 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:41,467 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 3ebb330a2ef5c6ab0e856e72db2c5e93, disabling compactions & flushes 2024-11-07T17:18:41,467 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:18:41,467 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:18:41,467 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. after waiting 0 ms 2024-11-07T17:18:41,467 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:18:41,467 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:18:41,467 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3ebb330a2ef5c6ab0e856e72db2c5e93: 2024-11-07T17:18:41,469 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:41,469 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1730999921469"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999921469"}]},"ts":"1730999921469"} 2024-11-07T17:18:41,472 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:41,473 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:41,474 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999921473"}]},"ts":"1730999921473"} 2024-11-07T17:18:41,476 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:41,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3ebb330a2ef5c6ab0e856e72db2c5e93, ASSIGN}] 2024-11-07T17:18:41,483 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3ebb330a2ef5c6ab0e856e72db2c5e93, ASSIGN 2024-11-07T17:18:41,484 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3ebb330a2ef5c6ab0e856e72db2c5e93, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:41,496 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:18:41,635 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=3ebb330a2ef5c6ab0e856e72db2c5e93, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:41,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 3ebb330a2ef5c6ab0e856e72db2c5e93, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:41,790 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:41,796 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:18:41,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => 3ebb330a2ef5c6ab0e856e72db2c5e93, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:41,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:18:41,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:41,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for 3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:18:41,797 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for 3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:18:41,799 INFO [StoreOpener-3ebb330a2ef5c6ab0e856e72db2c5e93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:18:41,801 INFO [StoreOpener-3ebb330a2ef5c6ab0e856e72db2c5e93-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ebb330a2ef5c6ab0e856e72db2c5e93 columnFamilyName testFamily 2024-11-07T17:18:41,801 DEBUG [StoreOpener-3ebb330a2ef5c6ab0e856e72db2c5e93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:41,802 INFO [StoreOpener-3ebb330a2ef5c6ab0e856e72db2c5e93-1 {}] regionserver.HStore(327): Store=3ebb330a2ef5c6ab0e856e72db2c5e93/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:41,803 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:18:41,803 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:18:41,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for 3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:18:41,808 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:41,809 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened 3ebb330a2ef5c6ab0e856e72db2c5e93; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69058048, jitterRate=0.02904510498046875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:41,810 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for 3ebb330a2ef5c6ab0e856e72db2c5e93: 2024-11-07T17:18:41,811 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93., pid=32, masterSystemTime=1730999921790 2024-11-07T17:18:41,814 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:18:41,814 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:18:41,814 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=3ebb330a2ef5c6ab0e856e72db2c5e93, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:41,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T17:18:41,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 3ebb330a2ef5c6ab0e856e72db2c5e93, server=3a0fde618c86,38099,1730999843825 in 180 msec 2024-11-07T17:18:41,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T17:18:41,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3ebb330a2ef5c6ab0e856e72db2c5e93, ASSIGN in 339 msec 2024-11-07T17:18:41,823 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:41,824 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999921823"}]},"ts":"1730999921823"} 2024-11-07T17:18:41,825 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:41,828 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:41,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 791 msec 2024-11-07T17:18:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T17:18:42,044 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-07T17:18:42,044 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:42,044 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:42,049 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:42,049 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:42,050 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-07T17:18:42,052 DEBUG [hconnection-0x5e77844a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:42,054 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47272, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:42,063 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:42,066 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:18:42,067 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938296 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:18:42,068 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T17:18:43,206 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:18:43,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T17:18:43,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:43,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-07T17:18:43,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:43,589 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-07T17:18:43,590 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-07T17:18:48,569 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-07T17:18:48,569 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d2a3f0e to 127.0.0.1:57710 2024-11-07T17:18:48,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:48,569 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:48,569 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:48,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:48,570 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4768f27f to 127.0.0.1:57710 2024-11-07T17:18:48,579 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 228), OpenFileDescriptor=454 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=321 (was 341), ProcessCount=11 (was 11), AvailableMemoryMB=3843 (was 3953) 2024-11-07T17:18:48,588 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=321, ProcessCount=11, AvailableMemoryMB=3843 2024-11-07T17:18:48,590 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56697735 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f6ff187 2024-11-07T17:18:48,593 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x56697735 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6f32bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:48,594 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x299dd6bd to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25f8da2f 2024-11-07T17:18:48,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3835c17a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:48,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-07T17:18:48,602 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:48,602 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:48,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-07T17:18:48,604 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T17:18:48,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741853_1029 (size=421) 2024-11-07T17:18:49,014 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f0dc1621293458c24d1a65e07708fda5, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:49,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741854_1030 (size=72) 2024-11-07T17:18:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing f0dc1621293458c24d1a65e07708fda5, disabling compactions & flushes 2024-11-07T17:18:49,423 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:18:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:18:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. after waiting 0 ms 2024-11-07T17:18:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:18:49,423 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:18:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for f0dc1621293458c24d1a65e07708fda5: 2024-11-07T17:18:49,425 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:49,425 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1730999929425"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999929425"}]},"ts":"1730999929425"} 2024-11-07T17:18:49,428 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:49,429 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:49,429 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999929429"}]},"ts":"1730999929429"} 2024-11-07T17:18:49,431 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:49,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=f0dc1621293458c24d1a65e07708fda5, ASSIGN}] 2024-11-07T17:18:49,438 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=f0dc1621293458c24d1a65e07708fda5, ASSIGN 2024-11-07T17:18:49,439 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=f0dc1621293458c24d1a65e07708fda5, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:49,589 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=f0dc1621293458c24d1a65e07708fda5, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:49,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure f0dc1621293458c24d1a65e07708fda5, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T17:18:49,745 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:49,750 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:18:49,750 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => f0dc1621293458c24d1a65e07708fda5, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:49,751 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:18:49,751 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:49,751 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:18:49,751 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:18:49,753 INFO [StoreOpener-f0dc1621293458c24d1a65e07708fda5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:18:49,755 INFO [StoreOpener-f0dc1621293458c24d1a65e07708fda5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0dc1621293458c24d1a65e07708fda5 columnFamilyName testFamily 2024-11-07T17:18:49,755 DEBUG [StoreOpener-f0dc1621293458c24d1a65e07708fda5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:49,756 INFO [StoreOpener-f0dc1621293458c24d1a65e07708fda5-1 {}] regionserver.HStore(327): Store=f0dc1621293458c24d1a65e07708fda5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:49,757 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:18:49,758 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:18:49,760 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:18:49,763 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:49,763 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened f0dc1621293458c24d1a65e07708fda5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58842928, jitterRate=-0.12317204475402832}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:49,764 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for f0dc1621293458c24d1a65e07708fda5: 2024-11-07T17:18:49,765 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5., pid=35, masterSystemTime=1730999929745 2024-11-07T17:18:49,767 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:18:49,768 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:18:49,768 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=f0dc1621293458c24d1a65e07708fda5, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:49,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-07T17:18:49,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure f0dc1621293458c24d1a65e07708fda5, server=3a0fde618c86,38099,1730999843825 in 177 msec 2024-11-07T17:18:49,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T17:18:49,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=f0dc1621293458c24d1a65e07708fda5, ASSIGN in 337 msec 2024-11-07T17:18:49,776 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:49,776 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999929776"}]},"ts":"1730999929776"} 2024-11-07T17:18:49,778 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:49,781 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:49,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 1.1810 sec 2024-11-07T17:18:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T17:18:51,608 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-07T17:18:51,608 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:51,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:51,612 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:51,612 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:51,613 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-07T17:18:51,615 DEBUG [hconnection-0x5a0b4f3e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:51,616 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53968, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:51,626 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:51,629 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:18:51,631 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938300 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:18:51,631 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T17:18:51,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:18:53,132 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1503 2024-11-07T17:18:53,132 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x299dd6bd to 127.0.0.1:57710 2024-11-07T17:18:53,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:53,132 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:53,133 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:53,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:53,133 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x56697735 to 127.0.0.1:57710 2024-11-07T17:18:53,142 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=454 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 321), ProcessCount=11 (was 11), AvailableMemoryMB=3784 (was 3843) 2024-11-07T17:18:53,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=3783 2024-11-07T17:18:53,153 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6fb2ce5b to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56c2363b 2024-11-07T17:18:53,156 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x6fb2ce5b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@381f296e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:53,157 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29e7df4f to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c48d5a4 2024-11-07T17:18:53,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65710060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:53,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-07T17:18:53,165 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:53,165 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:53,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-07T17:18:53,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T17:18:53,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741855_1031 (size=420) 2024-11-07T17:18:53,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-07T17:18:53,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:18:53,578 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 04ecb69da5f585e0f5d77714cd8dd665, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:53,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741856_1032 (size=71) 2024-11-07T17:18:53,986 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:53,986 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 04ecb69da5f585e0f5d77714cd8dd665, disabling compactions & flushes 2024-11-07T17:18:53,986 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:18:53,986 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:18:53,986 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. after waiting 0 ms 2024-11-07T17:18:53,986 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:18:53,986 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:18:53,986 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 04ecb69da5f585e0f5d77714cd8dd665: 2024-11-07T17:18:53,988 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:53,988 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730999933988"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999933988"}]},"ts":"1730999933988"} 2024-11-07T17:18:53,990 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:53,991 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:53,991 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999933991"}]},"ts":"1730999933991"} 2024-11-07T17:18:53,993 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:53,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=04ecb69da5f585e0f5d77714cd8dd665, ASSIGN}] 2024-11-07T17:18:54,000 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=04ecb69da5f585e0f5d77714cd8dd665, ASSIGN 2024-11-07T17:18:54,000 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=04ecb69da5f585e0f5d77714cd8dd665, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:54,151 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=04ecb69da5f585e0f5d77714cd8dd665, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:54,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 04ecb69da5f585e0f5d77714cd8dd665, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T17:18:54,306 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:54,311 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:18:54,311 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 04ecb69da5f585e0f5d77714cd8dd665, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:54,311 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:18:54,311 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:54,311 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:18:54,311 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:18:54,313 INFO [StoreOpener-04ecb69da5f585e0f5d77714cd8dd665-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:18:54,315 INFO [StoreOpener-04ecb69da5f585e0f5d77714cd8dd665-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04ecb69da5f585e0f5d77714cd8dd665 columnFamilyName testFamily 2024-11-07T17:18:54,315 DEBUG [StoreOpener-04ecb69da5f585e0f5d77714cd8dd665-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:54,316 INFO [StoreOpener-04ecb69da5f585e0f5d77714cd8dd665-1 {}] regionserver.HStore(327): Store=04ecb69da5f585e0f5d77714cd8dd665/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:54,317 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:18:54,317 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:18:54,319 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:18:54,322 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:54,323 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 04ecb69da5f585e0f5d77714cd8dd665; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70073633, jitterRate=0.04417850077152252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:54,323 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 04ecb69da5f585e0f5d77714cd8dd665: 2024-11-07T17:18:54,324 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665., pid=38, masterSystemTime=1730999934306 2024-11-07T17:18:54,326 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:18:54,326 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:18:54,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=04ecb69da5f585e0f5d77714cd8dd665, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:54,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T17:18:54,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 04ecb69da5f585e0f5d77714cd8dd665, server=3a0fde618c86,38099,1730999843825 in 176 msec 2024-11-07T17:18:54,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T17:18:54,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=04ecb69da5f585e0f5d77714cd8dd665, ASSIGN in 332 msec 2024-11-07T17:18:54,334 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:54,334 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999934334"}]},"ts":"1730999934334"} 2024-11-07T17:18:54,335 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:54,338 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:54,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 1.1750 sec 2024-11-07T17:18:55,273 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:18:55,592 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-07T17:18:55,593 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-07T17:18:56,079 DEBUG [master/3a0fde618c86:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T17:18:56,084 DEBUG [master/3a0fde618c86:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 306dab2b507c8881dd78df8ac96ce52f changed from -1.0 to 0.0, refreshing cache 2024-11-07T17:18:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T17:18:56,170 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-07T17:18:56,170 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:18:56,170 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:56,175 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T17:18:56,175 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:18:56,175 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-07T17:18:56,177 DEBUG [hconnection-0x26f56cd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:56,179 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53970, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:56,188 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:18:56,193 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68c52a5f to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72f683b9 2024-11-07T17:18:56,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f826340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:56,197 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:18:56,198 DEBUG [hconnection-0x5a8fcc67-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:18:56,200 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53980, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:18:56,204 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:18:56,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938305 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:56,205 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:18:56,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938305 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:18:56,206 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T17:18:56,224 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:18:56,974 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:34871,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:18:57,212 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T17:18:57.211Z, RpcRetryingCaller{globalStartTime=2024-11-07T17:18:56.206Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T17:18:57.211Z, RpcRetryingCaller{globalStartTime=2024-11-07T17:18:56.206Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:18:57,212 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1006 2024-11-07T17:18:57,212 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68c52a5f to 127.0.0.1:57710 2024-11-07T17:18:57,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:57,213 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29e7df4f to 127.0.0.1:57710 2024-11-07T17:18:57,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:57,213 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:18:57,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:18:57,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:18:57,213 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6fb2ce5b to 127.0.0.1:57710 2024-11-07T17:18:57,223 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 231) - Thread LEAK? -, OpenFileDescriptor=459 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=287 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=3731 (was 3783) 2024-11-07T17:18:57,231 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=287, ProcessCount=11, AvailableMemoryMB=3731 2024-11-07T17:18:57,233 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7764c92a to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2eb4061c 2024-11-07T17:18:57,236 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x7764c92a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb8f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:57,237 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cf1ac65 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7cf99b2d 2024-11-07T17:18:57,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d053001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:18:57,242 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:18:57,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T17:18:57,245 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:18:57,245 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:57,245 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-07T17:18:57,246 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:18:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T17:18:57,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741857_1033 (size=422) 2024-11-07T17:18:57,656 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 870ca7c659bec2ef0749d0a83c8f8581, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:18:57,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741858_1034 (size=73) 2024-11-07T17:18:57,707 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:18:58,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:58,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 870ca7c659bec2ef0749d0a83c8f8581, disabling compactions & flushes 2024-11-07T17:18:58,065 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:18:58,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:18:58,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. after waiting 0 ms 2024-11-07T17:18:58,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:18:58,065 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:18:58,066 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 870ca7c659bec2ef0749d0a83c8f8581: 2024-11-07T17:18:58,067 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:18:58,067 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1730999938067"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999938067"}]},"ts":"1730999938067"} 2024-11-07T17:18:58,069 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:18:58,070 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:18:58,071 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999938070"}]},"ts":"1730999938070"} 2024-11-07T17:18:58,072 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:18:58,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=870ca7c659bec2ef0749d0a83c8f8581, ASSIGN}] 2024-11-07T17:18:58,077 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=870ca7c659bec2ef0749d0a83c8f8581, ASSIGN 2024-11-07T17:18:58,078 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=870ca7c659bec2ef0749d0a83c8f8581, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:18:58,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=870ca7c659bec2ef0749d0a83c8f8581, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:58,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 870ca7c659bec2ef0749d0a83c8f8581, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:18:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T17:18:58,385 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:18:58,390 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:18:58,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 870ca7c659bec2ef0749d0a83c8f8581, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:18:58,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:18:58,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:18:58,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:18:58,391 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:18:58,394 INFO [StoreOpener-870ca7c659bec2ef0749d0a83c8f8581-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:18:58,395 INFO [StoreOpener-870ca7c659bec2ef0749d0a83c8f8581-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 870ca7c659bec2ef0749d0a83c8f8581 columnFamilyName testFamily 2024-11-07T17:18:58,396 DEBUG [StoreOpener-870ca7c659bec2ef0749d0a83c8f8581-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:18:58,396 INFO [StoreOpener-870ca7c659bec2ef0749d0a83c8f8581-1 {}] regionserver.HStore(327): Store=870ca7c659bec2ef0749d0a83c8f8581/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:18:58,397 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:18:58,398 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:18:58,400 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:18:58,403 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:18:58,404 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 870ca7c659bec2ef0749d0a83c8f8581; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72667260, jitterRate=0.08282655477523804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:18:58,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 870ca7c659bec2ef0749d0a83c8f8581: 2024-11-07T17:18:58,406 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581., pid=41, masterSystemTime=1730999938385 2024-11-07T17:18:58,408 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:18:58,408 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:18:58,408 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=870ca7c659bec2ef0749d0a83c8f8581, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:18:58,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T17:18:58,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 870ca7c659bec2ef0749d0a83c8f8581, server=3a0fde618c86,38099,1730999843825 in 179 msec 2024-11-07T17:18:58,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-07T17:18:58,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=870ca7c659bec2ef0749d0a83c8f8581, ASSIGN in 336 msec 2024-11-07T17:18:58,415 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:18:58,415 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999938415"}]},"ts":"1730999938415"} 2024-11-07T17:18:58,417 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:18:58,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:18:58,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.1780 sec 2024-11-07T17:19:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T17:19:00,250 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-07T17:19:00,251 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:19:00,251 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:00,256 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T17:19:00,256 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:00,256 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T17:19:00,258 DEBUG [hconnection-0x160d2d2e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:00,260 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55682, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:00,269 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:19:00,271 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:19:00,273 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:19:00,274 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T17:19:01,594 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-07T17:19:03,892 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:19:03,925 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-07T17:19:03,925 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:03,925 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T17:19:03,925 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T17:19:07,289 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:19:07,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:55682 deadline: 1730999949273, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:19:07,290 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:19:07,290 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:19:07,291 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7019 2024-11-07T17:19:07,291 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0cf1ac65 to 127.0.0.1:57710 2024-11-07T17:19:07,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:07,291 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:19:07,292 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:19:07,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:07,292 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7764c92a to 127.0.0.1:57710 2024-11-07T17:19:07,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=231 (was 236), OpenFileDescriptor=450 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=273 (was 287), ProcessCount=11 (was 11), AvailableMemoryMB=3647 (was 3731) 2024-11-07T17:19:07,309 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=3647 2024-11-07T17:19:07,311 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x773b38c7 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@696afa5a 2024-11-07T17:19:07,315 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x773b38c7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29917bd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:07,316 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x762af7e8 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7cef7242 2024-11-07T17:19:07,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523c9018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:07,321 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:19:07,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T17:19:07,324 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:19:07,324 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:07,324 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-07T17:19:07,324 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:19:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T17:19:07,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741859_1035 (size=428) 2024-11-07T17:19:07,734 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bf4cf17a5def170b7036786fc0b11c2e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:19:07,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741860_1036 (size=79) 2024-11-07T17:19:08,105 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613356 expired clientIPAndPort=172.17.0.2:35638, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:19:08,142 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:08,142 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing bf4cf17a5def170b7036786fc0b11c2e, disabling compactions & flushes 2024-11-07T17:19:08,142 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:19:08,142 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:19:08,143 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. after waiting 0 ms 2024-11-07T17:19:08,143 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:19:08,143 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:19:08,143 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for bf4cf17a5def170b7036786fc0b11c2e: 2024-11-07T17:19:08,144 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:19:08,144 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1730999948144"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999948144"}]},"ts":"1730999948144"} 2024-11-07T17:19:08,146 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:19:08,147 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:19:08,148 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999948147"}]},"ts":"1730999948147"} 2024-11-07T17:19:08,149 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:19:08,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=bf4cf17a5def170b7036786fc0b11c2e, ASSIGN}] 2024-11-07T17:19:08,155 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=bf4cf17a5def170b7036786fc0b11c2e, ASSIGN 2024-11-07T17:19:08,156 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=bf4cf17a5def170b7036786fc0b11c2e, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:19:08,306 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=bf4cf17a5def170b7036786fc0b11c2e, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:08,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure bf4cf17a5def170b7036786fc0b11c2e, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:19:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T17:19:08,461 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:19:08,466 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:19:08,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => bf4cf17a5def170b7036786fc0b11c2e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:19:08,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:19:08,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:08,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:19:08,467 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:19:08,469 INFO [StoreOpener-bf4cf17a5def170b7036786fc0b11c2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:19:08,471 INFO [StoreOpener-bf4cf17a5def170b7036786fc0b11c2e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf4cf17a5def170b7036786fc0b11c2e columnFamilyName testFamily 2024-11-07T17:19:08,471 DEBUG [StoreOpener-bf4cf17a5def170b7036786fc0b11c2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:08,471 INFO [StoreOpener-bf4cf17a5def170b7036786fc0b11c2e-1 {}] regionserver.HStore(327): Store=bf4cf17a5def170b7036786fc0b11c2e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:19:08,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:19:08,473 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:19:08,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:19:08,478 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:19:08,479 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened bf4cf17a5def170b7036786fc0b11c2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67377585, jitterRate=0.004004254937171936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:19:08,480 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for bf4cf17a5def170b7036786fc0b11c2e: 2024-11-07T17:19:08,481 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., pid=44, masterSystemTime=1730999948460 2024-11-07T17:19:08,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:19:08,483 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:19:08,483 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=bf4cf17a5def170b7036786fc0b11c2e, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:08,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-07T17:19:08,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure bf4cf17a5def170b7036786fc0b11c2e, server=3a0fde618c86,38099,1730999843825 in 177 msec 2024-11-07T17:19:08,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T17:19:08,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=bf4cf17a5def170b7036786fc0b11c2e, ASSIGN in 334 msec 2024-11-07T17:19:08,491 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:19:08,491 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999948491"}]},"ts":"1730999948491"} 2024-11-07T17:19:08,493 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:19:08,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:19:08,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.1740 sec 2024-11-07T17:19:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T17:19:10,328 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-07T17:19:10,329 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:19:10,329 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:10,332 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T17:19:10,333 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:10,333 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T17:19:10,335 DEBUG [hconnection-0x7cb4f64f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:10,336 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49724, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:10,345 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:19:10,347 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:19:10,348 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:19:10,349 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x773b38c7 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:19:10,349 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x773b38c7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:10,350 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49738, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:10,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:19:10,353 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:19:13,969 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:19:13,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T17:19:13,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:14,001 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T17:19:15,111 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613354 expired clientIPAndPort=172.17.0.2:35638, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:19:16,362 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-07T17:19:16,362 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-07T17:19:16,362 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:19:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T17:19:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:19:17,376 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:19:19,352 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1972ms 2024-11-07T17:19:19,352 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1972ms 2024-11-07T17:19:19,352 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:19:19,352 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T17:19:16.363Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-07T17:19:19.352Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1972ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1972ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1972ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T17:19:19,354 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-07T17:19:19,354 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x762af7e8 to 127.0.0.1:57710 2024-11-07T17:19:19,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:19,354 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:19:19,354 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:19:19,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:19,354 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x773b38c7 to 127.0.0.1:57710 2024-11-07T17:19:19,364 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=227 (was 231), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 273), ProcessCount=10 (was 11), AvailableMemoryMB=3623 (was 3647) 2024-11-07T17:19:19,372 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=3623 2024-11-07T17:19:19,374 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65847777 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d62d82e 2024-11-07T17:19:19,378 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x65847777 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4851e95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:19,379 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x45b91d0a to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@708dd6dc 2024-11-07T17:19:19,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f43875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:19,384 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:19:19,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-07T17:19:19,386 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:19:19,387 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:19,387 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-07T17:19:19,387 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:19:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T17:19:19,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741861_1037 (size=416) 2024-11-07T17:19:19,798 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 24ee1f613f99094716faf4222a66f5b0, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:19:19,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741862_1038 (size=67) 2024-11-07T17:19:20,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:20,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 24ee1f613f99094716faf4222a66f5b0, disabling compactions & flushes 2024-11-07T17:19:20,207 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:19:20,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:19:20,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. after waiting 0 ms 2024-11-07T17:19:20,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:19:20,207 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:19:20,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 24ee1f613f99094716faf4222a66f5b0: 2024-11-07T17:19:20,209 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:19:20,209 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1730999960209"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999960209"}]},"ts":"1730999960209"} 2024-11-07T17:19:20,212 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:19:20,212 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:19:20,213 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999960213"}]},"ts":"1730999960213"} 2024-11-07T17:19:20,214 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:19:20,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=24ee1f613f99094716faf4222a66f5b0, ASSIGN}] 2024-11-07T17:19:20,219 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=24ee1f613f99094716faf4222a66f5b0, ASSIGN 2024-11-07T17:19:20,220 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=24ee1f613f99094716faf4222a66f5b0, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:19:20,371 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=24ee1f613f99094716faf4222a66f5b0, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:20,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 24ee1f613f99094716faf4222a66f5b0, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:19:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T17:19:20,525 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:19:20,531 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:19:20,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 24ee1f613f99094716faf4222a66f5b0, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:19:20,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:19:20,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:20,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:19:20,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:19:20,534 INFO [StoreOpener-24ee1f613f99094716faf4222a66f5b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:19:20,535 INFO [StoreOpener-24ee1f613f99094716faf4222a66f5b0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24ee1f613f99094716faf4222a66f5b0 columnFamilyName testFamily 2024-11-07T17:19:20,536 DEBUG [StoreOpener-24ee1f613f99094716faf4222a66f5b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:20,536 INFO [StoreOpener-24ee1f613f99094716faf4222a66f5b0-1 {}] regionserver.HStore(327): Store=24ee1f613f99094716faf4222a66f5b0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:19:20,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:19:20,538 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:19:20,540 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:19:20,543 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:19:20,543 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 24ee1f613f99094716faf4222a66f5b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60858079, jitterRate=-0.09314395487308502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:19:20,544 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 24ee1f613f99094716faf4222a66f5b0: 2024-11-07T17:19:20,545 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0., pid=47, masterSystemTime=1730999960525 2024-11-07T17:19:20,547 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:19:20,547 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:19:20,548 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=24ee1f613f99094716faf4222a66f5b0, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:20,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T17:19:20,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 24ee1f613f99094716faf4222a66f5b0, server=3a0fde618c86,38099,1730999843825 in 177 msec 2024-11-07T17:19:20,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-07T17:19:20,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=24ee1f613f99094716faf4222a66f5b0, ASSIGN in 333 msec 2024-11-07T17:19:20,554 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:19:20,554 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999960554"}]},"ts":"1730999960554"} 2024-11-07T17:19:20,556 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:19:20,558 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:19:20,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.1730 sec 2024-11-07T17:19:21,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:19:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T17:19:22,392 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-07T17:19:22,392 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:19:22,392 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:22,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T17:19:22,397 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:22,397 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-07T17:19:22,399 DEBUG [hconnection-0x79458f02-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:22,400 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41260, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:22,408 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:19:22,413 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4be60061 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a2f0601 2024-11-07T17:19:22,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276a6b75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:22,417 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:19:22,418 DEBUG [hconnection-0x45976e97-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:22,419 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41262, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:22,422 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:19:22,423 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938322 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:22,424 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:19:22,424 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938322 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:22,425 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T17:19:23,218 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613349 expired clientIPAndPort=172.17.0.2:42212, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:19:23,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-07T17:19:23,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:23,876 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:19:25,821 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613343 expired clientIPAndPort=172.17.0.2:34264, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:19:25,821 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613342 expired clientIPAndPort=172.17.0.2:34264, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:19:25,821 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613345 expired clientIPAndPort=172.17.0.2:34264, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:19:26,059 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:19:26,096 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-07T17:19:27,252 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:19:28,405 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:34871,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:19:28,432 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T17:19:28.431Z, RpcRetryingCaller{globalStartTime=2024-11-07T17:19:22.424Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6007ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T17:19:28.431Z, RpcRetryingCaller{globalStartTime=2024-11-07T17:19:22.424Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6007ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6007ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6007ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:19:28,432 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6008 2024-11-07T17:19:28,432 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4be60061 to 127.0.0.1:57710 2024-11-07T17:19:28,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:28,432 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x45b91d0a to 127.0.0.1:57710 2024-11-07T17:19:28,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:28,432 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:19:28,433 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:19:28,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:28,433 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x65847777 to 127.0.0.1:57710 2024-11-07T17:19:28,442 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=232 (was 227) - Thread LEAK? -, OpenFileDescriptor=451 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=3606 (was 3623) 2024-11-07T17:19:28,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=232, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=3606 2024-11-07T17:19:28,452 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5f436d5f to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@206f4311 2024-11-07T17:19:28,455 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x5f436d5f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553ec143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:28,456 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d05f0d5 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56ce5118 2024-11-07T17:19:28,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1223e70b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:28,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:19:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T17:19:28,465 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:19:28,465 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:28,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-07T17:19:28,466 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:19:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T17:19:28,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741863_1039 (size=432) 2024-11-07T17:19:28,878 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3c6049f523e74afcad4a9e1f1737d1f2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:19:28,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741864_1040 (size=83) 2024-11-07T17:19:28,925 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:19:29,286 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:29,286 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 3c6049f523e74afcad4a9e1f1737d1f2, disabling compactions & flushes 2024-11-07T17:19:29,286 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:19:29,286 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:19:29,286 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. after waiting 0 ms 2024-11-07T17:19:29,286 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:19:29,286 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:19:29,286 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3c6049f523e74afcad4a9e1f1737d1f2: 2024-11-07T17:19:29,287 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:19:29,288 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1730999969288"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999969288"}]},"ts":"1730999969288"} 2024-11-07T17:19:29,290 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:19:29,290 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:19:29,291 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999969290"}]},"ts":"1730999969290"} 2024-11-07T17:19:29,292 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T17:19:29,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=3c6049f523e74afcad4a9e1f1737d1f2, ASSIGN}] 2024-11-07T17:19:29,297 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=3c6049f523e74afcad4a9e1f1737d1f2, ASSIGN 2024-11-07T17:19:29,297 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=3c6049f523e74afcad4a9e1f1737d1f2, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:19:29,448 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=3c6049f523e74afcad4a9e1f1737d1f2, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:29,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure 3c6049f523e74afcad4a9e1f1737d1f2, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:19:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T17:19:29,602 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:19:29,607 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:19:29,607 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => 3c6049f523e74afcad4a9e1f1737d1f2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:19:29,607 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false 3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:19:29,607 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:29,607 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for 3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:19:29,607 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for 3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:19:29,609 INFO [StoreOpener-3c6049f523e74afcad4a9e1f1737d1f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:19:29,611 INFO [StoreOpener-3c6049f523e74afcad4a9e1f1737d1f2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c6049f523e74afcad4a9e1f1737d1f2 columnFamilyName testFamily 2024-11-07T17:19:29,611 DEBUG [StoreOpener-3c6049f523e74afcad4a9e1f1737d1f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:29,612 INFO [StoreOpener-3c6049f523e74afcad4a9e1f1737d1f2-1 {}] regionserver.HStore(327): Store=3c6049f523e74afcad4a9e1f1737d1f2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:19:29,613 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:19:29,613 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:19:29,615 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for 3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:19:29,618 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:19:29,618 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened 3c6049f523e74afcad4a9e1f1737d1f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62042870, jitterRate=-0.07548919320106506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:19:29,619 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for 3c6049f523e74afcad4a9e1f1737d1f2: 2024-11-07T17:19:29,620 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., pid=50, masterSystemTime=1730999969602 2024-11-07T17:19:29,622 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:19:29,622 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:19:29,623 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=3c6049f523e74afcad4a9e1f1737d1f2, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:29,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-07T17:19:29,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure 3c6049f523e74afcad4a9e1f1737d1f2, server=3a0fde618c86,38099,1730999843825 in 175 msec 2024-11-07T17:19:29,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T17:19:29,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=3c6049f523e74afcad4a9e1f1737d1f2, ASSIGN in 330 msec 2024-11-07T17:19:29,629 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:19:29,629 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999969629"}]},"ts":"1730999969629"} 2024-11-07T17:19:29,630 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T17:19:29,633 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:19:29,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.1700 sec 2024-11-07T17:19:30,925 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613336 expired clientIPAndPort=172.17.0.2:34280, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:19:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T17:19:31,470 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-07T17:19:31,470 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T17:19:31,470 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:31,474 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T17:19:31,474 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:31,474 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T17:19:31,476 DEBUG [hconnection-0x39bfca8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:31,478 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46216, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:31,486 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:19:31,488 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:19:31,489 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:19:31,490 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x5f436d5f {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:19:31,490 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x5f436d5f {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:31,491 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46220, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:19:31,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:19:32,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T17:19:32,502 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-07T17:19:32,502 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-07T17:19:32,502 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:19:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T17:19:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T17:19:33,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:19:33,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:19:34,491 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=973ms 2024-11-07T17:19:34,492 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=973ms 2024-11-07T17:19:34,492 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:19:34,492 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T17:19:32.502Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-07T17:19:34.492Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=973ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=973ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=976ms, rpcTimeout=973ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T17:19:34,493 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3004 2024-11-07T17:19:34,493 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d05f0d5 to 127.0.0.1:57710 2024-11-07T17:19:34,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:34,494 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:19:34,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:19:34,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:34,494 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5f436d5f to 127.0.0.1:57710 2024-11-07T17:19:34,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=231 (was 232), OpenFileDescriptor=448 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 216), ProcessCount=9 (was 11), AvailableMemoryMB=4648 (was 3606) - AvailableMemoryMB LEAK? - 2024-11-07T17:19:34,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=9, AvailableMemoryMB=4648 2024-11-07T17:19:34,513 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x36c002f4 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14b79d74 2024-11-07T17:19:34,517 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x36c002f4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0dcc53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:34,517 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d1bb60c to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d2a852 2024-11-07T17:19:34,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b3cd9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:34,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:19:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T17:19:34,525 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:19:34,525 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:34,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-07T17:19:34,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:19:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T17:19:34,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741865_1041 (size=436) 2024-11-07T17:19:34,937 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 30c491763cb4452515bafed78f8bde08, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:19:34,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741866_1042 (size=87) 2024-11-07T17:19:35,016 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:19:35,109 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:19:35,346 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:35,346 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 30c491763cb4452515bafed78f8bde08, disabling compactions & flushes 2024-11-07T17:19:35,346 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:19:35,346 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:19:35,346 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. after waiting 0 ms 2024-11-07T17:19:35,346 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:19:35,346 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:19:35,346 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 30c491763cb4452515bafed78f8bde08: 2024-11-07T17:19:35,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:19:35,348 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1730999975347"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999975347"}]},"ts":"1730999975347"} 2024-11-07T17:19:35,349 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:19:35,350 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:19:35,350 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999975350"}]},"ts":"1730999975350"} 2024-11-07T17:19:35,352 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:19:35,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=30c491763cb4452515bafed78f8bde08, ASSIGN}] 2024-11-07T17:19:35,364 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=30c491763cb4452515bafed78f8bde08, ASSIGN 2024-11-07T17:19:35,365 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=30c491763cb4452515bafed78f8bde08, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:19:35,429 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613331 expired clientIPAndPort=172.17.0.2:52186, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:19:35,515 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=30c491763cb4452515bafed78f8bde08, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:35,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 30c491763cb4452515bafed78f8bde08, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:19:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T17:19:35,669 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:19:35,675 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:19:35,675 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => 30c491763cb4452515bafed78f8bde08, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:19:35,675 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 30c491763cb4452515bafed78f8bde08 2024-11-07T17:19:35,676 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:35,676 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for 30c491763cb4452515bafed78f8bde08 2024-11-07T17:19:35,676 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for 30c491763cb4452515bafed78f8bde08 2024-11-07T17:19:35,677 INFO [StoreOpener-30c491763cb4452515bafed78f8bde08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 30c491763cb4452515bafed78f8bde08 2024-11-07T17:19:35,679 INFO [StoreOpener-30c491763cb4452515bafed78f8bde08-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30c491763cb4452515bafed78f8bde08 columnFamilyName testFamily 2024-11-07T17:19:35,679 DEBUG [StoreOpener-30c491763cb4452515bafed78f8bde08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:35,680 INFO [StoreOpener-30c491763cb4452515bafed78f8bde08-1 {}] regionserver.HStore(327): Store=30c491763cb4452515bafed78f8bde08/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:19:35,681 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08 2024-11-07T17:19:35,681 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08 2024-11-07T17:19:35,684 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for 30c491763cb4452515bafed78f8bde08 2024-11-07T17:19:35,686 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:19:35,687 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened 30c491763cb4452515bafed78f8bde08; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66829549, jitterRate=-0.004162117838859558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:19:35,688 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for 30c491763cb4452515bafed78f8bde08: 2024-11-07T17:19:35,688 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08., pid=53, masterSystemTime=1730999975669 2024-11-07T17:19:35,690 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:19:35,690 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:19:35,691 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=30c491763cb4452515bafed78f8bde08, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:35,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T17:19:35,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 30c491763cb4452515bafed78f8bde08, server=3a0fde618c86,38099,1730999843825 in 175 msec 2024-11-07T17:19:35,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-07T17:19:35,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=30c491763cb4452515bafed78f8bde08, ASSIGN in 332 msec 2024-11-07T17:19:35,697 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:19:35,697 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999975697"}]},"ts":"1730999975697"} 2024-11-07T17:19:35,699 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:19:35,703 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:19:35,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.1800 sec 2024-11-07T17:19:37,431 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613329 expired clientIPAndPort=172.17.0.2:52186, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:19:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T17:19:37,530 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-07T17:19:37,530 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:19:37,530 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:37,534 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T17:19:37,535 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:37,535 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-07T17:19:37,537 DEBUG [hconnection-0x7be8cea-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:37,538 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46228, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:37,546 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:19:37,548 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T17:19:37,548 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:19:37,549 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x36c002f4 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:19:37,550 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x36c002f4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:37,551 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46234, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:19:37,553 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T17:19:37,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938334 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:37,554 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T17:19:37,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938334 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:37,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613282 2024-11-07T17:19:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46234 deadline: 1730999980553, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:19:37,555 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938334 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:19:37,556 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T17:19:37,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938335 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:37,557 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T17:19:37,557 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:19:37,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:19:37,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:37,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613280 2024-11-07T17:19:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46234 deadline: 1730999980557, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:19:37,559 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938336 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:19:37,560 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:19:37,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938337 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:37,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5792281934352613279 2024-11-07T17:19:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46234 deadline: 1730999980559, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:19:37,561 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938337 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:19:37,562 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:19:37,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938338 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:37,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5792281934352613278 2024-11-07T17:19:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46234 deadline: 1730999980561, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:19:37,563 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938338 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:19:37,564 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:19:37,564 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d1bb60c to 127.0.0.1:57710 2024-11-07T17:19:37,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:37,564 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:19:37,564 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:19:37,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:37,565 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x36c002f4 to 127.0.0.1:57710 2024-11-07T17:19:37,575 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=9 (was 9), AvailableMemoryMB=4625 (was 4648) 2024-11-07T17:19:37,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=9, AvailableMemoryMB=4625 2024-11-07T17:19:37,585 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b88043d to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5de73439 2024-11-07T17:19:37,589 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x0b88043d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e999e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:37,590 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46e87ba6 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1751b6d2 2024-11-07T17:19:37,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb3e1e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:37,596 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:19:37,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T17:19:37,598 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:19:37,598 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:37,599 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:19:37,599 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-07T17:19:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T17:19:37,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741867_1043 (size=422) 2024-11-07T17:19:38,009 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1df55e795a9b31b92ef935ae8ab2e252, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:19:38,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741868_1044 (size=73) 2024-11-07T17:19:38,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-07T17:19:38,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:38,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 1df55e795a9b31b92ef935ae8ab2e252, disabling compactions & flushes 2024-11-07T17:19:38,418 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:19:38,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:19:38,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. after waiting 0 ms 2024-11-07T17:19:38,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:19:38,418 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:19:38,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1df55e795a9b31b92ef935ae8ab2e252: 2024-11-07T17:19:38,419 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:19:38,419 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1730999978419"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999978419"}]},"ts":"1730999978419"} 2024-11-07T17:19:38,421 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:19:38,422 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:19:38,422 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999978422"}]},"ts":"1730999978422"} 2024-11-07T17:19:38,424 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:19:38,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=1df55e795a9b31b92ef935ae8ab2e252, ASSIGN}] 2024-11-07T17:19:38,429 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=1df55e795a9b31b92ef935ae8ab2e252, ASSIGN 2024-11-07T17:19:38,430 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=1df55e795a9b31b92ef935ae8ab2e252, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:19:38,580 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=1df55e795a9b31b92ef935ae8ab2e252, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:38,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure 1df55e795a9b31b92ef935ae8ab2e252, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:19:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T17:19:38,734 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:19:38,739 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:19:38,739 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => 1df55e795a9b31b92ef935ae8ab2e252, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:19:38,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true 1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:19:38,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:38,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for 1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:19:38,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for 1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:19:38,742 INFO [StoreOpener-1df55e795a9b31b92ef935ae8ab2e252-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:19:38,743 INFO [StoreOpener-1df55e795a9b31b92ef935ae8ab2e252-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1df55e795a9b31b92ef935ae8ab2e252 columnFamilyName testFamily 2024-11-07T17:19:38,743 DEBUG [StoreOpener-1df55e795a9b31b92ef935ae8ab2e252-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:38,744 INFO [StoreOpener-1df55e795a9b31b92ef935ae8ab2e252-1 {}] regionserver.HStore(327): Store=1df55e795a9b31b92ef935ae8ab2e252/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:19:38,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:19:38,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:19:38,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for 1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:19:38,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:19:38,751 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened 1df55e795a9b31b92ef935ae8ab2e252; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65851999, jitterRate=-0.018728747963905334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:19:38,752 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for 1df55e795a9b31b92ef935ae8ab2e252: 2024-11-07T17:19:38,753 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252., pid=56, masterSystemTime=1730999978734 2024-11-07T17:19:38,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:19:38,754 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:19:38,755 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=1df55e795a9b31b92ef935ae8ab2e252, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:38,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-07T17:19:38,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure 1df55e795a9b31b92ef935ae8ab2e252, server=3a0fde618c86,38099,1730999843825 in 175 msec 2024-11-07T17:19:38,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T17:19:38,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=1df55e795a9b31b92ef935ae8ab2e252, ASSIGN in 331 msec 2024-11-07T17:19:38,761 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:19:38,761 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999978761"}]},"ts":"1730999978761"} 2024-11-07T17:19:38,762 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:19:38,765 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:19:38,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1690 sec 2024-11-07T17:19:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T17:19:40,604 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-07T17:19:40,604 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:19:40,604 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:40,607 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T17:19:40,607 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:40,608 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T17:19:40,610 DEBUG [hconnection-0x6423eebe-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:40,611 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57730, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:40,618 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:19:40,620 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:19:40,621 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:19:41,177 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:19:41,534 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613324 expired clientIPAndPort=172.17.0.2:52204, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:19:43,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T17:19:43,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:43,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T17:19:43,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:44,100 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-07T17:19:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T17:19:47,626 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:19:49,622 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11007: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11007: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:19:49,622 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T17:19:49.622Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11007: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11007: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1995ms, rpcTimeout=1995ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:19:49,623 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9002 2024-11-07T17:19:49,624 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x46e87ba6 to 127.0.0.1:57710 2024-11-07T17:19:49,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:49,624 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:19:49,624 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:19:49,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:19:49,624 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b88043d to 127.0.0.1:57710 2024-11-07T17:19:49,633 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=227 (was 233), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 183), ProcessCount=9 (was 9), AvailableMemoryMB=4614 (was 4625) 2024-11-07T17:19:49,642 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=9, AvailableMemoryMB=4614 2024-11-07T17:19:49,643 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b7eb9a9 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76a4e040 2024-11-07T17:19:49,647 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x5b7eb9a9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fdd3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:49,648 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51690e43 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47bdcd8b 2024-11-07T17:19:49,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a08724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:19:49,654 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:19:49,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T17:19:49,656 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:19:49,656 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:49,656 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-07T17:19:49,657 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:19:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T17:19:49,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741869_1045 (size=420) 2024-11-07T17:19:50,068 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => faf897218923774aeeb6fe4badf399ad, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:19:50,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741870_1046 (size=71) 2024-11-07T17:19:50,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:50,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing faf897218923774aeeb6fe4badf399ad, disabling compactions & flushes 2024-11-07T17:19:50,477 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:19:50,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:19:50,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. after waiting 0 ms 2024-11-07T17:19:50,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:19:50,477 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:19:50,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for faf897218923774aeeb6fe4badf399ad: 2024-11-07T17:19:50,479 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:19:50,479 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1730999990479"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730999990479"}]},"ts":"1730999990479"} 2024-11-07T17:19:50,480 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:19:50,481 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:19:50,481 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999990481"}]},"ts":"1730999990481"} 2024-11-07T17:19:50,483 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:19:50,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=faf897218923774aeeb6fe4badf399ad, ASSIGN}] 2024-11-07T17:19:50,488 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=faf897218923774aeeb6fe4badf399ad, ASSIGN 2024-11-07T17:19:50,488 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=faf897218923774aeeb6fe4badf399ad, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:19:50,639 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=faf897218923774aeeb6fe4badf399ad, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:50,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure faf897218923774aeeb6fe4badf399ad, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:19:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T17:19:50,793 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:19:50,798 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:19:50,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => faf897218923774aeeb6fe4badf399ad, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:19:50,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true faf897218923774aeeb6fe4badf399ad 2024-11-07T17:19:50,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:19:50,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for faf897218923774aeeb6fe4badf399ad 2024-11-07T17:19:50,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for faf897218923774aeeb6fe4badf399ad 2024-11-07T17:19:50,801 INFO [StoreOpener-faf897218923774aeeb6fe4badf399ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region faf897218923774aeeb6fe4badf399ad 2024-11-07T17:19:50,803 INFO [StoreOpener-faf897218923774aeeb6fe4badf399ad-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region faf897218923774aeeb6fe4badf399ad columnFamilyName testFamily 2024-11-07T17:19:50,803 DEBUG [StoreOpener-faf897218923774aeeb6fe4badf399ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:19:50,803 INFO [StoreOpener-faf897218923774aeeb6fe4badf399ad-1 {}] regionserver.HStore(327): Store=faf897218923774aeeb6fe4badf399ad/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:19:50,804 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad 2024-11-07T17:19:50,805 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad 2024-11-07T17:19:50,807 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for faf897218923774aeeb6fe4badf399ad 2024-11-07T17:19:50,810 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:19:50,810 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened faf897218923774aeeb6fe4badf399ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58868840, jitterRate=-0.12278592586517334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:19:50,811 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for faf897218923774aeeb6fe4badf399ad: 2024-11-07T17:19:50,812 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad., pid=59, masterSystemTime=1730999990793 2024-11-07T17:19:50,814 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:19:50,814 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:19:50,814 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=faf897218923774aeeb6fe4badf399ad, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:19:50,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T17:19:50,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure faf897218923774aeeb6fe4badf399ad, server=3a0fde618c86,38099,1730999843825 in 175 msec 2024-11-07T17:19:50,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T17:19:50,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=faf897218923774aeeb6fe4badf399ad, ASSIGN in 331 msec 2024-11-07T17:19:50,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:19:50,820 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730999990820"}]},"ts":"1730999990820"} 2024-11-07T17:19:50,821 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:19:50,824 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:19:50,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 1.1700 sec 2024-11-07T17:19:51,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:19:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T17:19:52,661 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-07T17:19:52,661 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:19:52,661 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:52,665 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T17:19:52,665 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:19:52,665 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T17:19:52,667 DEBUG [hconnection-0x65d9b94d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:52,669 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60372, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:52,676 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:19:52,678 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:19:52,679 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:19:52,680 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x5b7eb9a9 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:19:52,680 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x5b7eb9a9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:19:52,681 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60384, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:19:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:19:52,684 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:19:52,684 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938350 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:52,685 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:19:52,685 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938350 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:19:52,685 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T17:19:53,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T17:19:53,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:19:54,127 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:19:56,335 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:19:56,374 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-07T17:19:57,747 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613311 expired clientIPAndPort=172.17.0.2:53980, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:19:58,473 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:19:59,620 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:34871,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:20:01,692 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9007 2024-11-07T17:20:01,692 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x51690e43 to 127.0.0.1:57710 2024-11-07T17:20:01,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:01,693 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:01,692 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938350 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:20:01,693 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:01,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:01,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613266' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T17:20:01,693 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5b7eb9a9 to 127.0.0.1:57710 2024-11-07T17:20:01,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60384 deadline: 1731000007692, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613266' does not exist 2024-11-07T17:20:01,693 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:20:01,693 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a0fde618c86,38099,1730999843825 for closing scanner id = -5792281934352613266 for faf897218923774aeeb6fe4badf399ad of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T17:20:01,703 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=227 (was 227), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=120 (was 142), ProcessCount=9 (was 9), AvailableMemoryMB=4579 (was 4614) 2024-11-07T17:20:01,712 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=120, ProcessCount=9, AvailableMemoryMB=4579 2024-11-07T17:20:01,714 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x446e3668 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7df75d71 2024-11-07T17:20:01,718 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x446e3668 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197272dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:01,719 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1eae5238 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@cb7ad92 2024-11-07T17:20:01,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f171a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:01,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T17:20:01,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:01,726 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:01,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-07T17:20:01,727 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T17:20:01,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741871_1047 (size=431) 2024-11-07T17:20:02,141 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 838a2de6cc4e0e72361bdedb53d1df08, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:02,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741872_1048 (size=82) 2024-11-07T17:20:02,186 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:20:02,549 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:02,549 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 838a2de6cc4e0e72361bdedb53d1df08, disabling compactions & flushes 2024-11-07T17:20:02,549 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:20:02,549 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:20:02,549 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. after waiting 0 ms 2024-11-07T17:20:02,549 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:20:02,549 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:20:02,549 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 838a2de6cc4e0e72361bdedb53d1df08: 2024-11-07T17:20:02,550 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:02,550 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731000002550"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000002550"}]},"ts":"1731000002550"} 2024-11-07T17:20:02,552 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:02,553 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:02,553 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000002553"}]},"ts":"1731000002553"} 2024-11-07T17:20:02,554 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:02,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=838a2de6cc4e0e72361bdedb53d1df08, ASSIGN}] 2024-11-07T17:20:02,559 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=838a2de6cc4e0e72361bdedb53d1df08, ASSIGN 2024-11-07T17:20:02,560 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=838a2de6cc4e0e72361bdedb53d1df08, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:02,710 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=838a2de6cc4e0e72361bdedb53d1df08, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:02,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 838a2de6cc4e0e72361bdedb53d1df08, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T17:20:02,864 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:02,870 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:20:02,871 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 838a2de6cc4e0e72361bdedb53d1df08, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:02,871 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:20:02,871 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:02,872 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:20:02,872 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:20:02,873 INFO [StoreOpener-838a2de6cc4e0e72361bdedb53d1df08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:20:02,875 INFO [StoreOpener-838a2de6cc4e0e72361bdedb53d1df08-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 838a2de6cc4e0e72361bdedb53d1df08 columnFamilyName testFamily 2024-11-07T17:20:02,875 DEBUG [StoreOpener-838a2de6cc4e0e72361bdedb53d1df08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:02,876 INFO [StoreOpener-838a2de6cc4e0e72361bdedb53d1df08-1 {}] regionserver.HStore(327): Store=838a2de6cc4e0e72361bdedb53d1df08/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:02,877 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:20:02,877 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:20:02,880 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:20:02,882 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:02,883 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 838a2de6cc4e0e72361bdedb53d1df08; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68648258, jitterRate=0.02293875813484192}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:02,884 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 838a2de6cc4e0e72361bdedb53d1df08: 2024-11-07T17:20:02,885 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08., pid=62, masterSystemTime=1731000002864 2024-11-07T17:20:02,886 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:20:02,886 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:20:02,887 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=838a2de6cc4e0e72361bdedb53d1df08, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:02,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T17:20:02,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 838a2de6cc4e0e72361bdedb53d1df08, server=3a0fde618c86,38099,1730999843825 in 176 msec 2024-11-07T17:20:02,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T17:20:02,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=838a2de6cc4e0e72361bdedb53d1df08, ASSIGN in 333 msec 2024-11-07T17:20:02,893 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:02,893 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000002893"}]},"ts":"1731000002893"} 2024-11-07T17:20:02,894 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:02,897 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:02,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.1730 sec 2024-11-07T17:20:03,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T17:20:03,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T17:20:04,731 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-07T17:20:04,732 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:04,732 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:04,736 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:04,736 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:04,736 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-07T17:20:04,739 DEBUG [hconnection-0x621bbbb0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:04,740 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37126, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:04,748 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:04,750 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T17:20:04,751 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T17:20:04,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938354 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:04,753 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T17:20:04,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938354 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:04,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613262 2024-11-07T17:20:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37126 deadline: 1731000007754, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:20:04,755 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:20:04,756 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T17:20:04,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938355 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:04,757 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T17:20:04,758 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:20:04,759 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:20:04,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938356 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:04,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-5792281934352613260 2024-11-07T17:20:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37126 deadline: 1731000007759, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:20:04,760 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:20:04,761 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:20:04,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938357 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:04,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-5792281934352613259 2024-11-07T17:20:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37126 deadline: 1731000007762, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:20:04,763 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:20:04,764 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:20:04,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938358 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:04,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-5792281934352613258 2024-11-07T17:20:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37126 deadline: 1731000007764, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T17:20:04,765 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:20:04,766 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T17:20:04,767 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1eae5238 to 127.0.0.1:57710 2024-11-07T17:20:04,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:04,767 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:04,767 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:04,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:04,767 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x446e3668 to 127.0.0.1:57710 2024-11-07T17:20:04,777 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=235 (was 227) - Thread LEAK? -, OpenFileDescriptor=450 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=118 (was 120), ProcessCount=9 (was 9), AvailableMemoryMB=4564 (was 4579) 2024-11-07T17:20:04,785 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=235, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=118, ProcessCount=9, AvailableMemoryMB=4563 2024-11-07T17:20:04,787 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x683af7cb to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c980e99 2024-11-07T17:20:04,791 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x683af7cb {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17742a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:04,792 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1233cc50 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b9a91e6 2024-11-07T17:20:04,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a1a7f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:04,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T17:20:04,800 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:04,800 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:04,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-07T17:20:04,801 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T17:20:04,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741873_1049 (size=425) 2024-11-07T17:20:05,211 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 72e185fd4f182be69856ff4c9c6c7377, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:05,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741874_1050 (size=76) 2024-11-07T17:20:05,619 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:05,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 72e185fd4f182be69856ff4c9c6c7377, disabling compactions & flushes 2024-11-07T17:20:05,620 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:20:05,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:20:05,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. after waiting 0 ms 2024-11-07T17:20:05,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:20:05,620 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:20:05,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 72e185fd4f182be69856ff4c9c6c7377: 2024-11-07T17:20:05,622 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:05,622 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731000005622"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000005622"}]},"ts":"1731000005622"} 2024-11-07T17:20:05,624 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:05,624 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:05,624 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000005624"}]},"ts":"1731000005624"} 2024-11-07T17:20:05,626 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:05,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=72e185fd4f182be69856ff4c9c6c7377, ASSIGN}] 2024-11-07T17:20:05,631 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=72e185fd4f182be69856ff4c9c6c7377, ASSIGN 2024-11-07T17:20:05,632 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=72e185fd4f182be69856ff4c9c6c7377, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:05,783 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=72e185fd4f182be69856ff4c9c6c7377, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:05,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure 72e185fd4f182be69856ff4c9c6c7377, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T17:20:05,938 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:05,943 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:20:05,943 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => 72e185fd4f182be69856ff4c9c6c7377, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:05,943 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true 72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:20:05,943 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:05,944 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for 72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:20:05,944 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for 72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:20:05,945 INFO [StoreOpener-72e185fd4f182be69856ff4c9c6c7377-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:20:05,947 INFO [StoreOpener-72e185fd4f182be69856ff4c9c6c7377-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72e185fd4f182be69856ff4c9c6c7377 columnFamilyName testFamily 2024-11-07T17:20:05,947 DEBUG [StoreOpener-72e185fd4f182be69856ff4c9c6c7377-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:05,948 INFO [StoreOpener-72e185fd4f182be69856ff4c9c6c7377-1 {}] regionserver.HStore(327): Store=72e185fd4f182be69856ff4c9c6c7377/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:05,949 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:20:05,949 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:20:05,951 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for 72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:20:05,954 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:05,954 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened 72e185fd4f182be69856ff4c9c6c7377; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70442865, jitterRate=0.04968048632144928}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:05,956 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for 72e185fd4f182be69856ff4c9c6c7377: 2024-11-07T17:20:05,957 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377., pid=65, masterSystemTime=1731000005937 2024-11-07T17:20:05,959 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:20:05,959 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:20:05,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=72e185fd4f182be69856ff4c9c6c7377, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:05,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T17:20:05,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure 72e185fd4f182be69856ff4c9c6c7377, server=3a0fde618c86,38099,1730999843825 in 176 msec 2024-11-07T17:20:05,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T17:20:05,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=72e185fd4f182be69856ff4c9c6c7377, ASSIGN in 332 msec 2024-11-07T17:20:05,964 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:05,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000005965"}]},"ts":"1731000005965"} 2024-11-07T17:20:05,966 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:05,970 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:05,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.1720 sec 2024-11-07T17:20:07,355 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613307 expired clientIPAndPort=172.17.0.2:55682, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:20:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T17:20:07,805 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-07T17:20:07,805 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:07,805 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:07,809 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:07,809 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:07,809 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T17:20:07,811 DEBUG [hconnection-0x62a19958-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:07,812 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37136, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:07,819 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:07,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:20:07,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:20:07,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T17:20:08,416 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:20:08,462 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-07T17:20:08,462 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-07T17:20:09,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T17:20:09,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:20:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37136 deadline: 1731000010822, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:20:09,841 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:20:09,841 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938363 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:20:09,842 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2021 2024-11-07T17:20:09,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1233cc50 to 127.0.0.1:57710 2024-11-07T17:20:09,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:09,843 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:09,843 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:09,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:09,843 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x683af7cb to 127.0.0.1:57710 2024-11-07T17:20:09,852 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234 (was 235), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=109 (was 118), ProcessCount=9 (was 9), AvailableMemoryMB=4529 (was 4563) 2024-11-07T17:20:09,860 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=234, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=109, ProcessCount=9, AvailableMemoryMB=4528 2024-11-07T17:20:09,862 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03cf3254 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@296a71bf 2024-11-07T17:20:09,865 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x03cf3254 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f2b30c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:09,866 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1a8753bb to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5377cf12 2024-11-07T17:20:09,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29501aba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:09,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T17:20:09,873 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:09,874 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:09,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-07T17:20:09,874 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T17:20:09,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741875_1051 (size=426) 2024-11-07T17:20:10,284 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 63b8b46898cf84e2969939becdc3d6cb, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:10,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741876_1052 (size=77) 2024-11-07T17:20:10,691 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:10,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 63b8b46898cf84e2969939becdc3d6cb, disabling compactions & flushes 2024-11-07T17:20:10,692 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:20:10,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:20:10,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. after waiting 0 ms 2024-11-07T17:20:10,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:20:10,692 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:20:10,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 63b8b46898cf84e2969939becdc3d6cb: 2024-11-07T17:20:10,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:10,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731000010693"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000010693"}]},"ts":"1731000010693"} 2024-11-07T17:20:10,695 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:10,696 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:10,696 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000010696"}]},"ts":"1731000010696"} 2024-11-07T17:20:10,697 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:10,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=63b8b46898cf84e2969939becdc3d6cb, ASSIGN}] 2024-11-07T17:20:10,703 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=63b8b46898cf84e2969939becdc3d6cb, ASSIGN 2024-11-07T17:20:10,704 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=63b8b46898cf84e2969939becdc3d6cb, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:10,855 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=63b8b46898cf84e2969939becdc3d6cb, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:10,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 63b8b46898cf84e2969939becdc3d6cb, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T17:20:11,008 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:11,013 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:20:11,013 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 63b8b46898cf84e2969939becdc3d6cb, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:11,014 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:20:11,014 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:11,014 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:20:11,014 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:20:11,016 INFO [StoreOpener-63b8b46898cf84e2969939becdc3d6cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:20:11,018 INFO [StoreOpener-63b8b46898cf84e2969939becdc3d6cb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63b8b46898cf84e2969939becdc3d6cb columnFamilyName testFamily 2024-11-07T17:20:11,018 DEBUG [StoreOpener-63b8b46898cf84e2969939becdc3d6cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:11,018 INFO [StoreOpener-63b8b46898cf84e2969939becdc3d6cb-1 {}] regionserver.HStore(327): Store=63b8b46898cf84e2969939becdc3d6cb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:11,019 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:20:11,019 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:20:11,022 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:20:11,024 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:11,025 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 63b8b46898cf84e2969939becdc3d6cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73256781, jitterRate=0.0916111022233963}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:11,025 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 63b8b46898cf84e2969939becdc3d6cb: 2024-11-07T17:20:11,026 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., pid=68, masterSystemTime=1731000011008 2024-11-07T17:20:11,028 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:20:11,028 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:20:11,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=63b8b46898cf84e2969939becdc3d6cb, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:11,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-07T17:20:11,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 63b8b46898cf84e2969939becdc3d6cb, server=3a0fde618c86,38099,1730999843825 in 174 msec 2024-11-07T17:20:11,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T17:20:11,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=63b8b46898cf84e2969939becdc3d6cb, ASSIGN in 329 msec 2024-11-07T17:20:11,033 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:11,033 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000011033"}]},"ts":"1731000011033"} 2024-11-07T17:20:11,035 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:11,038 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:11,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1660 sec 2024-11-07T17:20:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T17:20:12,878 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-07T17:20:12,878 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:12,878 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:12,881 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:12,882 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:12,882 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T17:20:12,884 DEBUG [hconnection-0x37b9f87e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:12,885 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57066, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:12,892 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:12,894 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:20:12,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:20:13,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T17:20:13,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:13,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T17:20:13,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T17:20:14,464 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-07T17:20:14,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:20:15,902 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5022: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5022: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:20:15,903 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T17:20:15.902Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5022: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5022: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:20:15,903 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3009 2024-11-07T17:20:15,903 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1a8753bb to 127.0.0.1:57710 2024-11-07T17:20:15,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:15,904 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:15,904 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:15,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:15,904 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x03cf3254 to 127.0.0.1:57710 2024-11-07T17:20:15,913 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=231 (was 234), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=100 (was 109), ProcessCount=9 (was 9), AvailableMemoryMB=4522 (was 4528) 2024-11-07T17:20:15,921 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=100, ProcessCount=9, AvailableMemoryMB=4522 2024-11-07T17:20:15,923 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f44ba3f to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38f13fdc 2024-11-07T17:20:15,926 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x2f44ba3f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8623148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:15,927 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5198b778 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1383693d 2024-11-07T17:20:15,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@199cbc30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:15,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T17:20:15,934 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:15,934 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:15,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-07T17:20:15,935 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T17:20:15,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741877_1053 (size=424) 2024-11-07T17:20:16,344 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b3be295a4062dec176f6add711e118b7, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:16,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741878_1054 (size=75) 2024-11-07T17:20:16,411 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:20:16,515 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:20:16,752 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:16,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing b3be295a4062dec176f6add711e118b7, disabling compactions & flushes 2024-11-07T17:20:16,753 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:20:16,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:20:16,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. after waiting 0 ms 2024-11-07T17:20:16,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:20:16,753 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:20:16,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for b3be295a4062dec176f6add711e118b7: 2024-11-07T17:20:16,754 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:16,754 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731000016754"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000016754"}]},"ts":"1731000016754"} 2024-11-07T17:20:16,756 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:16,757 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:16,757 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000016757"}]},"ts":"1731000016757"} 2024-11-07T17:20:16,758 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:16,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=b3be295a4062dec176f6add711e118b7, ASSIGN}] 2024-11-07T17:20:16,766 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=b3be295a4062dec176f6add711e118b7, ASSIGN 2024-11-07T17:20:16,767 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=b3be295a4062dec176f6add711e118b7, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:16,862 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613301 expired clientIPAndPort=172.17.0.2:49738, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:20:16,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=b3be295a4062dec176f6add711e118b7, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:16,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure b3be295a4062dec176f6add711e118b7, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T17:20:17,071 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:17,075 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:20:17,075 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => b3be295a4062dec176f6add711e118b7, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:17,076 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true b3be295a4062dec176f6add711e118b7 2024-11-07T17:20:17,076 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:17,076 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for b3be295a4062dec176f6add711e118b7 2024-11-07T17:20:17,076 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for b3be295a4062dec176f6add711e118b7 2024-11-07T17:20:17,078 INFO [StoreOpener-b3be295a4062dec176f6add711e118b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b3be295a4062dec176f6add711e118b7 2024-11-07T17:20:17,079 INFO [StoreOpener-b3be295a4062dec176f6add711e118b7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3be295a4062dec176f6add711e118b7 columnFamilyName testFamily 2024-11-07T17:20:17,080 DEBUG [StoreOpener-b3be295a4062dec176f6add711e118b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:17,080 INFO [StoreOpener-b3be295a4062dec176f6add711e118b7-1 {}] regionserver.HStore(327): Store=b3be295a4062dec176f6add711e118b7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:17,081 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7 2024-11-07T17:20:17,082 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7 2024-11-07T17:20:17,083 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for b3be295a4062dec176f6add711e118b7 2024-11-07T17:20:17,085 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:17,086 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened b3be295a4062dec176f6add711e118b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63447624, jitterRate=-0.05455672740936279}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:17,087 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for b3be295a4062dec176f6add711e118b7: 2024-11-07T17:20:17,088 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7., pid=71, masterSystemTime=1731000017071 2024-11-07T17:20:17,089 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:20:17,089 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:20:17,090 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=b3be295a4062dec176f6add711e118b7, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:17,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T17:20:17,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure b3be295a4062dec176f6add711e118b7, server=3a0fde618c86,38099,1730999843825 in 172 msec 2024-11-07T17:20:17,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-07T17:20:17,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=b3be295a4062dec176f6add711e118b7, ASSIGN in 328 msec 2024-11-07T17:20:17,095 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:17,095 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000017095"}]},"ts":"1731000017095"} 2024-11-07T17:20:17,097 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:17,100 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:17,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.1680 sec 2024-11-07T17:20:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T17:20:18,938 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-07T17:20:18,938 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:18,938 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:18,942 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:18,942 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:18,942 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T17:20:18,944 DEBUG [hconnection-0x485625a3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:18,945 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57072, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:18,952 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:18,954 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:20:18,954 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:20:18,955 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x2f44ba3f {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:20:18,956 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x2f44ba3f {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:18,957 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57084, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:18,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:20:18,959 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:20:18,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938375 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:18,959 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:20:18,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938375 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:18,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T17:20:20,465 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-07T17:20:21,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:20:21,962 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3002 2024-11-07T17:20:21,962 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5198b778 to 127.0.0.1:57710 2024-11-07T17:20:21,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:21,962 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938375 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:20:21,962 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:21,963 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:21,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:21,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613241' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T17:20:21,963 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f44ba3f to 127.0.0.1:57710 2024-11-07T17:20:21,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:57084 deadline: 1731000022962, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-5792281934352613241' does not exist 2024-11-07T17:20:21,963 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:20:21,963 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a0fde618c86,38099,1730999843825 for closing scanner id = -5792281934352613241 for b3be295a4062dec176f6add711e118b7 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T17:20:21,973 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=230 (was 231), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=92 (was 100), ProcessCount=9 (was 9), AvailableMemoryMB=4495 (was 4522) 2024-11-07T17:20:21,981 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=92, ProcessCount=9, AvailableMemoryMB=4494 2024-11-07T17:20:21,982 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a3339e5 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d41f25a 2024-11-07T17:20:21,986 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x0a3339e5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71182493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:21,987 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59b17c65 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67a65bdd 2024-11-07T17:20:21,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c9aadd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:21,992 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:21,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-07T17:20:21,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:21,994 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:21,994 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-07T17:20:21,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T17:20:22,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741879_1055 (size=416) 2024-11-07T17:20:22,406 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 88f460756d06deb6989559d688e71bad, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:22,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741880_1056 (size=67) 2024-11-07T17:20:22,460 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:20:22,578 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:20:22,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:22,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 88f460756d06deb6989559d688e71bad, disabling compactions & flushes 2024-11-07T17:20:22,816 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:20:22,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:20:22,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. after waiting 0 ms 2024-11-07T17:20:22,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:20:22,816 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:20:22,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 88f460756d06deb6989559d688e71bad: 2024-11-07T17:20:22,817 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:22,817 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731000022817"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000022817"}]},"ts":"1731000022817"} 2024-11-07T17:20:22,819 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:22,820 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:22,820 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000022820"}]},"ts":"1731000022820"} 2024-11-07T17:20:22,821 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:22,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=88f460756d06deb6989559d688e71bad, ASSIGN}] 2024-11-07T17:20:22,827 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=88f460756d06deb6989559d688e71bad, ASSIGN 2024-11-07T17:20:22,828 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=88f460756d06deb6989559d688e71bad, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:22,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=88f460756d06deb6989559d688e71bad, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:22,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 88f460756d06deb6989559d688e71bad, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T17:20:23,132 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:23,137 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:20:23,138 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 88f460756d06deb6989559d688e71bad, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:23,138 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 88f460756d06deb6989559d688e71bad 2024-11-07T17:20:23,138 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:23,138 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 88f460756d06deb6989559d688e71bad 2024-11-07T17:20:23,138 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 88f460756d06deb6989559d688e71bad 2024-11-07T17:20:23,140 INFO [StoreOpener-88f460756d06deb6989559d688e71bad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 88f460756d06deb6989559d688e71bad 2024-11-07T17:20:23,142 INFO [StoreOpener-88f460756d06deb6989559d688e71bad-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88f460756d06deb6989559d688e71bad columnFamilyName testFamily 2024-11-07T17:20:23,142 DEBUG [StoreOpener-88f460756d06deb6989559d688e71bad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:23,142 INFO [StoreOpener-88f460756d06deb6989559d688e71bad-1 {}] regionserver.HStore(327): Store=88f460756d06deb6989559d688e71bad/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:23,143 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad 2024-11-07T17:20:23,144 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad 2024-11-07T17:20:23,146 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 88f460756d06deb6989559d688e71bad 2024-11-07T17:20:23,148 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:23,148 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 88f460756d06deb6989559d688e71bad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61691546, jitterRate=-0.0807243287563324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:23,149 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 88f460756d06deb6989559d688e71bad: 2024-11-07T17:20:23,150 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad., pid=74, masterSystemTime=1731000023132 2024-11-07T17:20:23,152 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:20:23,152 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:20:23,152 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=88f460756d06deb6989559d688e71bad, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:23,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-07T17:20:23,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 88f460756d06deb6989559d688e71bad, server=3a0fde618c86,38099,1730999843825 in 174 msec 2024-11-07T17:20:23,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T17:20:23,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=88f460756d06deb6989559d688e71bad, ASSIGN in 329 msec 2024-11-07T17:20:23,157 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:23,157 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000023157"}]},"ts":"1731000023157"} 2024-11-07T17:20:23,158 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:23,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:23,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.1700 sec 2024-11-07T17:20:23,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T17:20:23,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:23,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-07T17:20:23,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:23,968 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613299 expired clientIPAndPort=172.17.0.2:49738, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:20:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T17:20:24,998 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-07T17:20:24,998 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:24,998 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:25,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-07T17:20:25,005 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938376 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:25,006 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:25,006 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:25,006 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-07T17:20:25,008 DEBUG [hconnection-0x7c965c11-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:25,010 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37990, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:25,017 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:25,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:20:25,020 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938379 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:20:25,020 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T17:20:26,466 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-07T17:20:28,640 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:20:28,972 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613294 expired clientIPAndPort=172.17.0.2:41262, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:20:31,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-07T17:20:31,521 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59b17c65 to 127.0.0.1:57710 2024-11-07T17:20:31,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:31,522 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:31,522 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:31,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:31,522 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a3339e5 to 127.0.0.1:57710 2024-11-07T17:20:31,531 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=230 (was 230), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=86 (was 92), ProcessCount=9 (was 9), AvailableMemoryMB=4433 (was 4494) 2024-11-07T17:20:31,539 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=86, ProcessCount=9, AvailableMemoryMB=4432 2024-11-07T17:20:31,541 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x569f21a9 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27b6853d 2024-11-07T17:20:31,545 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x569f21a9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547913da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:31,546 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7eed5e87 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33746f2a 2024-11-07T17:20:31,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc98284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:31,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-07T17:20:31,554 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:31,554 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:31,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-07T17:20:31,555 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T17:20:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741881_1057 (size=420) 2024-11-07T17:20:31,734 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:20:31,964 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => dc1045097a0a1d6b49a4a2d49b1a4b05, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:31,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741882_1058 (size=71) 2024-11-07T17:20:32,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:32,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing dc1045097a0a1d6b49a4a2d49b1a4b05, disabling compactions & flushes 2024-11-07T17:20:32,372 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:20:32,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:20:32,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. after waiting 0 ms 2024-11-07T17:20:32,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:20:32,372 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:20:32,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for dc1045097a0a1d6b49a4a2d49b1a4b05: 2024-11-07T17:20:32,373 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:32,373 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731000032373"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000032373"}]},"ts":"1731000032373"} 2024-11-07T17:20:32,375 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:32,376 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:32,376 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000032376"}]},"ts":"1731000032376"} 2024-11-07T17:20:32,377 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:32,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dc1045097a0a1d6b49a4a2d49b1a4b05, ASSIGN}] 2024-11-07T17:20:32,383 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dc1045097a0a1d6b49a4a2d49b1a4b05, ASSIGN 2024-11-07T17:20:32,383 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dc1045097a0a1d6b49a4a2d49b1a4b05, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:32,534 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=dc1045097a0a1d6b49a4a2d49b1a4b05, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:32,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure dc1045097a0a1d6b49a4a2d49b1a4b05, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T17:20:32,688 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:32,693 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:20:32,693 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => dc1045097a0a1d6b49a4a2d49b1a4b05, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:32,693 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:20:32,693 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:32,693 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:20:32,693 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:20:32,695 INFO [StoreOpener-dc1045097a0a1d6b49a4a2d49b1a4b05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:20:32,697 INFO [StoreOpener-dc1045097a0a1d6b49a4a2d49b1a4b05-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc1045097a0a1d6b49a4a2d49b1a4b05 columnFamilyName testFamily 2024-11-07T17:20:32,697 DEBUG [StoreOpener-dc1045097a0a1d6b49a4a2d49b1a4b05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:32,698 INFO [StoreOpener-dc1045097a0a1d6b49a4a2d49b1a4b05-1 {}] regionserver.HStore(327): Store=dc1045097a0a1d6b49a4a2d49b1a4b05/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:32,699 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:20:32,699 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:20:32,701 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:20:32,704 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:32,704 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened dc1045097a0a1d6b49a4a2d49b1a4b05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68669882, jitterRate=0.02326098084449768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:32,705 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for dc1045097a0a1d6b49a4a2d49b1a4b05: 2024-11-07T17:20:32,706 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05., pid=77, masterSystemTime=1731000032688 2024-11-07T17:20:32,708 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:20:32,708 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:20:32,708 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=dc1045097a0a1d6b49a4a2d49b1a4b05, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:32,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T17:20:32,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure dc1045097a0a1d6b49a4a2d49b1a4b05, server=3a0fde618c86,38099,1730999843825 in 174 msec 2024-11-07T17:20:32,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-07T17:20:32,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dc1045097a0a1d6b49a4a2d49b1a4b05, ASSIGN in 329 msec 2024-11-07T17:20:32,714 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:32,714 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000032714"}]},"ts":"1731000032714"} 2024-11-07T17:20:32,716 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:32,719 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:32,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1670 sec 2024-11-07T17:20:32,884 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:34871,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:20:33,076 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613289 expired clientIPAndPort=172.17.0.2:46220, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:20:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-07T17:20:33,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T17:20:34,558 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-07T17:20:34,558 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:34,558 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:34,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-07T17:20:34,562 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938380 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:34,563 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:34,563 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:34,563 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-07T17:20:34,565 DEBUG [hconnection-0x7f239d59-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:34,566 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53460, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:34,573 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:34,575 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:20:34,576 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938383 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T17:20:34,577 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T17:20:35,078 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613287 expired clientIPAndPort=172.17.0.2:46220, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:20:36,077 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-07T17:20:36,078 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7eed5e87 to 127.0.0.1:57710 2024-11-07T17:20:36,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:36,078 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:36,078 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:36,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:36,078 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x569f21a9 to 127.0.0.1:57710 2024-11-07T17:20:36,087 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=231 (was 230) - Thread LEAK? -, OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=87 (was 86) - SystemLoadAverage LEAK? -, ProcessCount=9 (was 9), AvailableMemoryMB=4432 (was 4432) 2024-11-07T17:20:36,095 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=87, ProcessCount=9, AvailableMemoryMB=4429 2024-11-07T17:20:36,097 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fdf745d to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11a9829a 2024-11-07T17:20:36,100 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x1fdf745d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d93041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:36,101 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x316e5fad to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@384b6450 2024-11-07T17:20:36,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c528298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:36,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-07T17:20:36,111 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:36,111 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:36,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-07T17:20:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T17:20:36,112 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:36,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741883_1059 (size=419) 2024-11-07T17:20:36,524 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1b10e4be130aface5367bc50dee0bc3b, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:36,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741884_1060 (size=70) 2024-11-07T17:20:36,933 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:36,933 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 1b10e4be130aface5367bc50dee0bc3b, disabling compactions & flushes 2024-11-07T17:20:36,933 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:20:36,933 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:20:36,933 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. after waiting 0 ms 2024-11-07T17:20:36,933 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:20:36,933 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:20:36,933 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1b10e4be130aface5367bc50dee0bc3b: 2024-11-07T17:20:36,934 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:36,935 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731000036935"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000036935"}]},"ts":"1731000036935"} 2024-11-07T17:20:36,937 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:36,937 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:36,937 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000036937"}]},"ts":"1731000036937"} 2024-11-07T17:20:36,939 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:36,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=1b10e4be130aface5367bc50dee0bc3b, ASSIGN}] 2024-11-07T17:20:36,944 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=1b10e4be130aface5367bc50dee0bc3b, ASSIGN 2024-11-07T17:20:36,945 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=1b10e4be130aface5367bc50dee0bc3b, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:37,095 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=1b10e4be130aface5367bc50dee0bc3b, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:37,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 1b10e4be130aface5367bc50dee0bc3b, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T17:20:37,249 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:37,254 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:20:37,254 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => 1b10e4be130aface5367bc50dee0bc3b, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:37,255 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:20:37,255 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:37,255 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for 1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:20:37,255 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for 1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:20:37,257 INFO [StoreOpener-1b10e4be130aface5367bc50dee0bc3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:20:37,258 INFO [StoreOpener-1b10e4be130aface5367bc50dee0bc3b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b10e4be130aface5367bc50dee0bc3b columnFamilyName testFamily 2024-11-07T17:20:37,258 DEBUG [StoreOpener-1b10e4be130aface5367bc50dee0bc3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:37,259 INFO [StoreOpener-1b10e4be130aface5367bc50dee0bc3b-1 {}] regionserver.HStore(327): Store=1b10e4be130aface5367bc50dee0bc3b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:37,260 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:20:37,260 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:20:37,262 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for 1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:20:37,265 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:37,266 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened 1b10e4be130aface5367bc50dee0bc3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69148375, jitterRate=0.030391082167625427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:37,266 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for 1b10e4be130aface5367bc50dee0bc3b: 2024-11-07T17:20:37,267 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b., pid=80, masterSystemTime=1731000037249 2024-11-07T17:20:37,269 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:20:37,269 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:20:37,270 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=1b10e4be130aface5367bc50dee0bc3b, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:37,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-07T17:20:37,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 1b10e4be130aface5367bc50dee0bc3b, server=3a0fde618c86,38099,1730999843825 in 174 msec 2024-11-07T17:20:37,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T17:20:37,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=1b10e4be130aface5367bc50dee0bc3b, ASSIGN in 330 msec 2024-11-07T17:20:37,275 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:37,275 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000037275"}]},"ts":"1731000037275"} 2024-11-07T17:20:37,276 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:37,279 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:37,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1700 sec 2024-11-07T17:20:38,195 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:20:38,469 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-07T17:20:38,469 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-07T17:20:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T17:20:39,115 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-07T17:20:39,115 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:39,115 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:39,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-07T17:20:39,120 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938384 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:39,121 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:39,122 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:39,122 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-07T17:20:39,124 DEBUG [hconnection-0x7b70e829-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:39,125 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53476, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:39,133 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:39,135 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:20:39,137 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:20:39,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938387 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:39,138 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:20:39,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938387 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:39,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T17:20:42,141 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4005: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4005: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3003ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:20:42,142 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3003 2024-11-07T17:20:42,142 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x316e5fad to 127.0.0.1:57710 2024-11-07T17:20:42,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:42,142 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:42,142 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:42,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:42,143 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1fdf745d to 127.0.0.1:57710 2024-11-07T17:20:42,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=230 (was 231), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=80 (was 87), ProcessCount=9 (was 9), AvailableMemoryMB=4408 (was 4429) 2024-11-07T17:20:42,159 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=80, ProcessCount=9, AvailableMemoryMB=4408 2024-11-07T17:20:42,161 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e559cac to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6cdd6436 2024-11-07T17:20:42,193 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x2e559cac {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca43325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:42,194 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f0dde7c to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61728c6d 2024-11-07T17:20:42,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c0dc13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:42,200 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:42,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T17:20:42,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:42,202 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:42,202 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-07T17:20:42,203 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T17:20:42,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741885_1061 (size=421) 2024-11-07T17:20:42,612 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 611065383d17cc239f94ff70641327a5, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:42,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741886_1062 (size=72) 2024-11-07T17:20:42,639 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:20:43,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:43,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 611065383d17cc239f94ff70641327a5, disabling compactions & flushes 2024-11-07T17:20:43,020 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:20:43,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:20:43,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. after waiting 0 ms 2024-11-07T17:20:43,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:20:43,020 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:20:43,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 611065383d17cc239f94ff70641327a5: 2024-11-07T17:20:43,022 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:43,022 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731000043022"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000043022"}]},"ts":"1731000043022"} 2024-11-07T17:20:43,023 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:43,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:43,024 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000043024"}]},"ts":"1731000043024"} 2024-11-07T17:20:43,026 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:43,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=611065383d17cc239f94ff70641327a5, ASSIGN}] 2024-11-07T17:20:43,030 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=611065383d17cc239f94ff70641327a5, ASSIGN 2024-11-07T17:20:43,030 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=611065383d17cc239f94ff70641327a5, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:43,181 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=611065383d17cc239f94ff70641327a5, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:43,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 611065383d17cc239f94ff70641327a5, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T17:20:43,335 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:43,339 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:20:43,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => 611065383d17cc239f94ff70641327a5, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:43,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 611065383d17cc239f94ff70641327a5 2024-11-07T17:20:43,339 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:43,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for 611065383d17cc239f94ff70641327a5 2024-11-07T17:20:43,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for 611065383d17cc239f94ff70641327a5 2024-11-07T17:20:43,341 INFO [StoreOpener-611065383d17cc239f94ff70641327a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 611065383d17cc239f94ff70641327a5 2024-11-07T17:20:43,343 INFO [StoreOpener-611065383d17cc239f94ff70641327a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 611065383d17cc239f94ff70641327a5 columnFamilyName testFamily 2024-11-07T17:20:43,343 DEBUG [StoreOpener-611065383d17cc239f94ff70641327a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:43,344 INFO [StoreOpener-611065383d17cc239f94ff70641327a5-1 {}] regionserver.HStore(327): Store=611065383d17cc239f94ff70641327a5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:43,344 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5 2024-11-07T17:20:43,345 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5 2024-11-07T17:20:43,347 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for 611065383d17cc239f94ff70641327a5 2024-11-07T17:20:43,349 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:43,349 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened 611065383d17cc239f94ff70641327a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66580608, jitterRate=-0.007871627807617188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:43,350 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for 611065383d17cc239f94ff70641327a5: 2024-11-07T17:20:43,351 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5., pid=83, masterSystemTime=1731000043334 2024-11-07T17:20:43,352 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:20:43,352 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:20:43,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=611065383d17cc239f94ff70641327a5, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:43,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T17:20:43,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 611065383d17cc239f94ff70641327a5, server=3a0fde618c86,38099,1730999843825 in 171 msec 2024-11-07T17:20:43,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-07T17:20:43,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=611065383d17cc239f94ff70641327a5, ASSIGN in 326 msec 2024-11-07T17:20:43,357 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:43,357 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000043357"}]},"ts":"1731000043357"} 2024-11-07T17:20:43,358 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:43,361 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:43,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1610 sec 2024-11-07T17:20:43,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T17:20:43,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:43,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T17:20:43,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:44,471 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-07T17:20:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T17:20:45,206 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-07T17:20:45,206 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:45,206 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:45,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-07T17:20:45,210 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938388 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:45,212 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:45,212 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:45,212 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T17:20:45,214 DEBUG [hconnection-0x4ea16aab-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:45,215 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39912, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:45,222 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:45,224 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:20:45,229 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:20:45,229 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T17:20:47,192 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613273 expired clientIPAndPort=172.17.0.2:57730, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:20:48,873 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:20:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T17:20:51,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:20:52,243 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:20:52,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:39912 deadline: 1731000054228, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T17:20:52,244 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T17:20:52,245 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12654462139356938391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T17:20:52,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7021 2024-11-07T17:20:52,245 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f0dde7c to 127.0.0.1:57710 2024-11-07T17:20:52,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:52,246 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:20:52,246 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:20:52,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:20:52,246 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e559cac to 127.0.0.1:57710 2024-11-07T17:20:52,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230 (was 230), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=67 (was 80), ProcessCount=9 (was 9), AvailableMemoryMB=4352 (was 4408) 2024-11-07T17:20:52,263 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=67, ProcessCount=9, AvailableMemoryMB=4352 2024-11-07T17:20:52,264 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06f3dba9 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b24af1b 2024-11-07T17:20:52,269 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x06f3dba9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4167654a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:52,270 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19738ced to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@615bc1aa 2024-11-07T17:20:52,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796d70bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:20:52,275 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:20:52,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T17:20:52,276 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:20:52,276 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:52,276 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-07T17:20:52,277 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:20:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T17:20:52,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741887_1063 (size=427) 2024-11-07T17:20:52,686 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c3bf6516bafcafad3967a8a32e883ac6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:20:52,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741888_1064 (size=78) 2024-11-07T17:20:53,093 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:53,093 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing c3bf6516bafcafad3967a8a32e883ac6, disabling compactions & flushes 2024-11-07T17:20:53,093 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:20:53,093 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:20:53,093 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. after waiting 0 ms 2024-11-07T17:20:53,093 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:20:53,093 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:20:53,093 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for c3bf6516bafcafad3967a8a32e883ac6: 2024-11-07T17:20:53,094 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:20:53,095 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731000053094"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000053094"}]},"ts":"1731000053094"} 2024-11-07T17:20:53,096 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:20:53,097 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:20:53,097 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000053097"}]},"ts":"1731000053097"} 2024-11-07T17:20:53,099 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:20:53,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c3bf6516bafcafad3967a8a32e883ac6, ASSIGN}] 2024-11-07T17:20:53,104 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c3bf6516bafcafad3967a8a32e883ac6, ASSIGN 2024-11-07T17:20:53,104 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c3bf6516bafcafad3967a8a32e883ac6, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:20:53,255 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=c3bf6516bafcafad3967a8a32e883ac6, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:53,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure c3bf6516bafcafad3967a8a32e883ac6, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:20:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T17:20:53,409 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:20:53,414 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:20:53,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => c3bf6516bafcafad3967a8a32e883ac6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:20:53,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:20:53,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:20:53,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:20:53,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:20:53,417 INFO [StoreOpener-c3bf6516bafcafad3967a8a32e883ac6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:20:53,418 INFO [StoreOpener-c3bf6516bafcafad3967a8a32e883ac6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3bf6516bafcafad3967a8a32e883ac6 columnFamilyName testFamily 2024-11-07T17:20:53,418 DEBUG [StoreOpener-c3bf6516bafcafad3967a8a32e883ac6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:20:53,419 INFO [StoreOpener-c3bf6516bafcafad3967a8a32e883ac6-1 {}] regionserver.HStore(327): Store=c3bf6516bafcafad3967a8a32e883ac6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:20:53,420 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:20:53,420 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:20:53,423 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:20:53,425 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:20:53,425 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened c3bf6516bafcafad3967a8a32e883ac6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66473302, jitterRate=-0.009470611810684204}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:20:53,426 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for c3bf6516bafcafad3967a8a32e883ac6: 2024-11-07T17:20:53,427 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., pid=86, masterSystemTime=1731000053409 2024-11-07T17:20:53,428 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:20:53,428 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:20:53,429 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=c3bf6516bafcafad3967a8a32e883ac6, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:20:53,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-07T17:20:53,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure c3bf6516bafcafad3967a8a32e883ac6, server=3a0fde618c86,38099,1730999843825 in 173 msec 2024-11-07T17:20:53,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T17:20:53,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c3bf6516bafcafad3967a8a32e883ac6, ASSIGN in 328 msec 2024-11-07T17:20:53,433 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:20:53,434 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000053433"}]},"ts":"1731000053433"} 2024-11-07T17:20:53,435 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:20:53,438 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:20:53,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1630 sec 2024-11-07T17:20:53,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T17:20:53,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:20:54,197 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613271 expired clientIPAndPort=172.17.0.2:57730, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:20:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T17:20:55,290 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-07T17:20:55,290 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:20:55,291 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:55,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-07T17:20:55,295 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938395 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:20:55,295 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T17:20:55,296 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:20:55,296 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T17:20:55,297 DEBUG [hconnection-0x3dfd2722-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:55,298 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57700, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:55,305 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:20:55,307 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:20:55,307 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:20:55,308 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x06f3dba9 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:20:55,308 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x06f3dba9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:20:55,309 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57714, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:20:55,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:20:55,311 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:20:56,472 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T17:20:58,916 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:21:01,311 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms 2024-11-07T17:21:01,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms 2024-11-07T17:21:01,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:21:01,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T17:21:01,969 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:21:02,204 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613266 expired clientIPAndPort=172.17.0.2:60384, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:21:02,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:21:02,326 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T17:21:04,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1981ms 2024-11-07T17:21:04,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1981ms 2024-11-07T17:21:04,312 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:21:04,313 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T17:21:01.312Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms 2024-11-07T17:21:04.312Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1981ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1987ms, rpcTimeout=1981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T17:21:04,314 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-07T17:21:04,314 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x19738ced to 127.0.0.1:57710 2024-11-07T17:21:04,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:04,314 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:21:04,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:21:04,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:04,314 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x06f3dba9 to 127.0.0.1:57710 2024-11-07T17:21:04,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=226 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=67 (was 67), ProcessCount=9 (was 9), AvailableMemoryMB=4337 (was 4352) 2024-11-07T17:21:04,331 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=67, ProcessCount=9, AvailableMemoryMB=4337 2024-11-07T17:21:04,332 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e97173d to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3195f2ae 2024-11-07T17:21:04,340 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x7e97173d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac574f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:21:04,341 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3b3b8455 to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fe23cdf 2024-11-07T17:21:04,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698ae8e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:21:04,347 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:21:04,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-07T17:21:04,348 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:21:04,349 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:21:04,349 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-07T17:21:04,349 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:21:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T17:21:04,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741889_1065 (size=415) 2024-11-07T17:21:04,759 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f87acb04cfac5bbaea483f6e8cf81cd8, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:21:04,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741890_1066 (size=66) 2024-11-07T17:21:04,768 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:21:04,768 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f87acb04cfac5bbaea483f6e8cf81cd8, disabling compactions & flushes 2024-11-07T17:21:04,768 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:04,768 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:04,768 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. after waiting 0 ms 2024-11-07T17:21:04,768 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:04,768 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:04,768 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f87acb04cfac5bbaea483f6e8cf81cd8: 2024-11-07T17:21:04,769 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:21:04,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731000064769"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000064769"}]},"ts":"1731000064769"} 2024-11-07T17:21:04,771 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:21:04,771 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:21:04,771 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000064771"}]},"ts":"1731000064771"} 2024-11-07T17:21:04,773 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:21:04,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f87acb04cfac5bbaea483f6e8cf81cd8, ASSIGN}] 2024-11-07T17:21:04,779 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f87acb04cfac5bbaea483f6e8cf81cd8, ASSIGN 2024-11-07T17:21:04,779 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f87acb04cfac5bbaea483f6e8cf81cd8, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:21:04,806 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613262 expired clientIPAndPort=172.17.0.2:37126, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:04,806 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613260 expired clientIPAndPort=172.17.0.2:37126, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:04,806 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613259 expired clientIPAndPort=172.17.0.2:37126, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:04,930 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=f87acb04cfac5bbaea483f6e8cf81cd8, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:21:04,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure f87acb04cfac5bbaea483f6e8cf81cd8, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:21:05,084 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:21:05,089 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:05,089 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => f87acb04cfac5bbaea483f6e8cf81cd8, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:21:05,089 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:05,089 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:21:05,090 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:05,090 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:05,091 INFO [StoreOpener-f87acb04cfac5bbaea483f6e8cf81cd8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:05,093 INFO [StoreOpener-f87acb04cfac5bbaea483f6e8cf81cd8-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f87acb04cfac5bbaea483f6e8cf81cd8 columnFamilyName testFamily 2024-11-07T17:21:05,093 DEBUG [StoreOpener-f87acb04cfac5bbaea483f6e8cf81cd8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:21:05,093 INFO [StoreOpener-f87acb04cfac5bbaea483f6e8cf81cd8-1 {}] regionserver.HStore(327): Store=f87acb04cfac5bbaea483f6e8cf81cd8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:21:05,094 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:05,094 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:05,096 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:05,099 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:21:05,099 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened f87acb04cfac5bbaea483f6e8cf81cd8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66408350, jitterRate=-0.010438472032546997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:21:05,100 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for f87acb04cfac5bbaea483f6e8cf81cd8: 2024-11-07T17:21:05,100 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8., pid=89, masterSystemTime=1731000065084 2024-11-07T17:21:05,102 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:05,102 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:05,102 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=f87acb04cfac5bbaea483f6e8cf81cd8, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:21:05,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T17:21:05,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure f87acb04cfac5bbaea483f6e8cf81cd8, server=3a0fde618c86,38099,1730999843825 in 172 msec 2024-11-07T17:21:05,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-07T17:21:05,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f87acb04cfac5bbaea483f6e8cf81cd8, ASSIGN in 327 msec 2024-11-07T17:21:05,107 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:21:05,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000065107"}]},"ts":"1731000065107"} 2024-11-07T17:21:05,108 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:21:05,111 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:21:05,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 764 msec 2024-11-07T17:21:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T17:21:05,351 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-07T17:21:05,351 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:21:05,351 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:21:05,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-07T17:21:05,356 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938402 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:21:05,357 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T17:21:05,357 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:21:05,357 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-07T17:21:05,359 DEBUG [hconnection-0x445c98d5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:21:05,360 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33460, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:21:05,367 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:21:05,368 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T17:21:05,370 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T17:21:05,370 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938405 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:21:05,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T17:21:05,371 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938405 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:21:05,371 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T17:21:07,267 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:34871,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:21:08,473 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-07T17:21:08,826 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:21:09,910 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613253 expired clientIPAndPort=172.17.0.2:37136, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:21:10,624 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:21:13,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-07T17:21:13,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:21:14,381 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10018: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10018: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T17:21:14,382 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9010 2024-11-07T17:21:14,382 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3b3b8455 to 127.0.0.1:57710 2024-11-07T17:21:14,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:14,382 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:21:14,382 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:21:14,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:14,382 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e97173d to 127.0.0.1:57710 2024-11-07T17:21:14,391 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=229 (was 226) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=57 (was 67), ProcessCount=9 (was 9), AvailableMemoryMB=4465 (was 4337) - AvailableMemoryMB LEAK? - 2024-11-07T17:21:14,400 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=57, ProcessCount=9, AvailableMemoryMB=4464 2024-11-07T17:21:14,401 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x436590ad to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@233033fc 2024-11-07T17:21:14,406 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x436590ad {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a7198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:21:14,406 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ad46a0a to 127.0.0.1:57710 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6e4e284d 2024-11-07T17:21:14,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1413dd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T17:21:14,413 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613248 expired clientIPAndPort=172.17.0.2:57066, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:21:14,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T17:21:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T17:21:14,415 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T17:21:14,416 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:21:14,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-07T17:21:14,416 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T17:21:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T17:21:14,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741891_1067 (size=431) 2024-11-07T17:21:14,825 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0f511d49e0cd173770e983418883dc7d, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392 2024-11-07T17:21:14,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741892_1068 (size=82) 2024-11-07T17:21:14,872 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:21:15,232 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:21:15,232 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 0f511d49e0cd173770e983418883dc7d, disabling compactions & flushes 2024-11-07T17:21:15,232 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:15,232 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:15,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. after waiting 0 ms 2024-11-07T17:21:15,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:15,233 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:15,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0f511d49e0cd173770e983418883dc7d: 2024-11-07T17:21:15,234 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T17:21:15,234 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731000075234"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731000075234"}]},"ts":"1731000075234"} 2024-11-07T17:21:15,235 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T17:21:15,236 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T17:21:15,236 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000075236"}]},"ts":"1731000075236"} 2024-11-07T17:21:15,237 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T17:21:15,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0f511d49e0cd173770e983418883dc7d, ASSIGN}] 2024-11-07T17:21:15,242 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0f511d49e0cd173770e983418883dc7d, ASSIGN 2024-11-07T17:21:15,242 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0f511d49e0cd173770e983418883dc7d, ASSIGN; state=OFFLINE, location=3a0fde618c86,38099,1730999843825; forceNewPlan=false, retain=false 2024-11-07T17:21:15,393 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=0f511d49e0cd173770e983418883dc7d, regionState=OPENING, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:21:15,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 0f511d49e0cd173770e983418883dc7d, server=3a0fde618c86,38099,1730999843825}] 2024-11-07T17:21:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T17:21:15,547 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 3a0fde618c86,38099,1730999843825 2024-11-07T17:21:15,552 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:15,552 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 0f511d49e0cd173770e983418883dc7d, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T17:21:15,552 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:15,552 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T17:21:15,552 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:15,552 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:15,554 INFO [StoreOpener-0f511d49e0cd173770e983418883dc7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:15,555 INFO [StoreOpener-0f511d49e0cd173770e983418883dc7d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f511d49e0cd173770e983418883dc7d columnFamilyName testFamily 2024-11-07T17:21:15,555 DEBUG [StoreOpener-0f511d49e0cd173770e983418883dc7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T17:21:15,556 INFO [StoreOpener-0f511d49e0cd173770e983418883dc7d-1 {}] regionserver.HStore(327): Store=0f511d49e0cd173770e983418883dc7d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T17:21:15,556 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:15,557 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:15,559 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:15,561 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T17:21:15,562 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 0f511d49e0cd173770e983418883dc7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70972552, jitterRate=0.05757343769073486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T17:21:15,562 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 0f511d49e0cd173770e983418883dc7d: 2024-11-07T17:21:15,563 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., pid=92, masterSystemTime=1731000075546 2024-11-07T17:21:15,564 DEBUG [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:15,565 INFO [RS_OPEN_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:15,565 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=0f511d49e0cd173770e983418883dc7d, regionState=OPEN, openSeqNum=2, regionLocation=3a0fde618c86,38099,1730999843825 2024-11-07T17:21:15,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T17:21:15,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 0f511d49e0cd173770e983418883dc7d, server=3a0fde618c86,38099,1730999843825 in 171 msec 2024-11-07T17:21:15,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T17:21:15,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0f511d49e0cd173770e983418883dc7d, ASSIGN in 326 msec 2024-11-07T17:21:15,569 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T17:21:15,569 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731000075569"}]},"ts":"1731000075569"} 2024-11-07T17:21:15,570 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T17:21:15,573 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T17:21:15,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1590 sec 2024-11-07T17:21:16,415 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613246 expired clientIPAndPort=172.17.0.2:57066, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:21:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36799 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T17:21:17,420 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-07T17:21:17,420 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T17:21:17,420 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:21:17,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38099 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-07T17:21:17,424 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12654462139356938406 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T17:21:17,425 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T17:21:17,425 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T17:21:17,425 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T17:21:17,427 DEBUG [hconnection-0x4cc60c8b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:21:17,428 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60634, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:21:17,434 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T17:21:17,436 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T17:21:17,436 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T17:21:17,437 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x436590ad {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a0fde618c86,38099,1730999843825, seqNum=-1] 2024-11-07T17:21:17,438 DEBUG [ReadOnlyZKClient-127.0.0.1:57710@0x436590ad {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T17:21:17,439 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60640, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T17:21:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:21:17,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:21:18,442 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-07T17:21:18,442 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-07T17:21:18,442 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:21:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T17:21:19,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2] 2024-11-07T17:21:19,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T17:21:20,441 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=986ms, rpcTimeout=981ms 2024-11-07T17:21:20,441 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=986ms, rpcTimeout=981ms 2024-11-07T17:21:20,441 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., hostname=3a0fde618c86,38099,1730999843825, seqNum=2 from cache 2024-11-07T17:21:20,442 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T17:21:18.442Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-07T17:21:20.441Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=986ms, rpcTimeout=981ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a0fde618c86:38099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=986ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=986ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T17:21:20,443 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3006 2024-11-07T17:21:20,443 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ad46a0a to 127.0.0.1:57710 2024-11-07T17:21:20,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:20,443 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T17:21:20,443 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T17:21:20,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:20,443 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x436590ad to 127.0.0.1:57710 2024-11-07T17:21:20,452 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=231 (was 229) - Thread LEAK? -, OpenFileDescriptor=449 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=52 (was 57), ProcessCount=9 (was 9), AvailableMemoryMB=4461 (was 4464) 2024-11-07T17:21:20,452 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-07T17:21:20,453 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-07T17:21:20,453 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62f30d47 to 127.0.0.1:57710 2024-11-07T17:21:20,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:20,453 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T17:21:20,453 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=468212891, stopped=false 2024-11-07T17:21:20,453 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=3a0fde618c86,36799,1730999842914 2024-11-07T17:21:20,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T17:21:20,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T17:21:20,455 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-07T17:21:20,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:21:20,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:21:20,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:20,455 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '3a0fde618c86,38099,1730999843825' ***** 2024-11-07T17:21:20,456 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-07T17:21:20,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T17:21:20,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T17:21:20,456 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T17:21:20,456 INFO [RS:0;3a0fde618c86:38099 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T17:21:20,456 INFO [RS:0;3a0fde618c86:38099 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T17:21:20,456 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-07T17:21:20,456 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for bf4cf17a5def170b7036786fc0b11c2e 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 1b10e4be130aface5367bc50dee0bc3b 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 306dab2b507c8881dd78df8ac96ce52f 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for d19a44f348557665f53debac39ada87f 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 3c6049f523e74afcad4a9e1f1737d1f2 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 88f460756d06deb6989559d688e71bad 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for dc1045097a0a1d6b49a4a2d49b1a4b05 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 1df55e795a9b31b92ef935ae8ab2e252 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for a64612bdb26c784d77aca9c346e0664d 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 611065383d17cc239f94ff70641327a5 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 24ee1f613f99094716faf4222a66f5b0 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for b3be295a4062dec176f6add711e118b7 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 272802cb3d0cc73b8a0b7d1ab03b0b71 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 04ecb69da5f585e0f5d77714cd8dd665 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for c3bf6516bafcafad3967a8a32e883ac6 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for aa009bbf2796b670b2b6eb293cb23b7d 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 72e185fd4f182be69856ff4c9c6c7377 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for f87acb04cfac5bbaea483f6e8cf81cd8 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 3cfc81fcf468d7f0dd7b1f50158dd4fa 2024-11-07T17:21:20,457 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing bf4cf17a5def170b7036786fc0b11c2e, disabling compactions & flushes 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for f859399bd9a2dff5654f21dedb4a5651 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for f0dc1621293458c24d1a65e07708fda5 2024-11-07T17:21:20,457 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 838a2de6cc4e0e72361bdedb53d1df08 2024-11-07T17:21:20,457 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 30c491763cb4452515bafed78f8bde08 2024-11-07T17:21:20,457 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. after waiting 0 ms 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 3ebb330a2ef5c6ab0e856e72db2c5e93 2024-11-07T17:21:20,457 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 870ca7c659bec2ef0749d0a83c8f8581 2024-11-07T17:21:20,457 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 63b8b46898cf84e2969939becdc3d6cb 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 0f511d49e0cd173770e983418883dc7d 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1224): stopping server 3a0fde618c86,38099,1730999843825 2024-11-07T17:21:20,458 DEBUG [RS:0;3a0fde618c86:38099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-07T17:21:20,458 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing bf4cf17a5def170b7036786fc0b11c2e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:20,458 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-07T17:21:20,458 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T17:21:20,458 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T17:21:20,458 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T17:21:20,458 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T17:21:20,458 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T17:21:20,459 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-07T17:21:20,459 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1588230740, 1b10e4be130aface5367bc50dee0bc3b, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, bf4cf17a5def170b7036786fc0b11c2e, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:20,475 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T17:21:20,515 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e/.tmp/testFamily/132963cd983142f7aacff4dd96b93467 is 61, key is row-0/testFamily:testQualifier/1730999950338/Put/seqid=0 2024-11-07T17:21:20,522 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/.tmp/info/6f26fb95e1164ece9ed946608cc2e8a0 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71./info:regioninfo/1730999877765/Put/seqid=0 2024-11-07T17:21:20,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741893_1069 (size=5049) 2024-11-07T17:21:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741894_1070 (size=51873) 2024-11-07T17:21:20,659 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1588230740, 1b10e4be130aface5367bc50dee0bc3b, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, bf4cf17a5def170b7036786fc0b11c2e, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:20,859 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1588230740, 1b10e4be130aface5367bc50dee0bc3b, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, bf4cf17a5def170b7036786fc0b11c2e, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:20,926 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e/.tmp/testFamily/132963cd983142f7aacff4dd96b93467 2024-11-07T17:21:20,928 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/.tmp/info/6f26fb95e1164ece9ed946608cc2e8a0 2024-11-07T17:21:20,956 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T17:21:20,972 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e/.tmp/testFamily/132963cd983142f7aacff4dd96b93467 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e/testFamily/132963cd983142f7aacff4dd96b93467 2024-11-07T17:21:20,980 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e/testFamily/132963cd983142f7aacff4dd96b93467, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:20,983 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bf4cf17a5def170b7036786fc0b11c2e in 525ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:20,988 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/bf4cf17a5def170b7036786fc0b11c2e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:20,989 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/.tmp/table/0af2718d91ec476387e1b8ce6ebf916b is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1730999877778/Put/seqid=0 2024-11-07T17:21:20,991 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:21:20,991 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for bf4cf17a5def170b7036786fc0b11c2e: 2024-11-07T17:21:20,991 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1730999947321.bf4cf17a5def170b7036786fc0b11c2e. 2024-11-07T17:21:20,991 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1b10e4be130aface5367bc50dee0bc3b, disabling compactions & flushes 2024-11-07T17:21:20,991 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:21:20,991 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:21:20,992 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. after waiting 0 ms 2024-11-07T17:21:20,992 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:21:20,992 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1b10e4be130aface5367bc50dee0bc3b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:20,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741895_1071 (size=9821) 2024-11-07T17:21:21,009 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b/.tmp/testFamily/483ec65a242549b38fabbe1225deacfb is 61, key is row-0/testFamily:testQualifier/1731000039127/Put/seqid=0 2024-11-07T17:21:21,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741896_1072 (size=5049) 2024-11-07T17:21:21,053 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:21:21,060 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1588230740, 1b10e4be130aface5367bc50dee0bc3b, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:21,127 INFO [regionserver/3a0fde618c86:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T17:21:21,127 INFO [regionserver/3a0fde618c86:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T17:21:21,260 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1588230740, 1b10e4be130aface5367bc50dee0bc3b, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:21,395 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/.tmp/table/0af2718d91ec476387e1b8ce6ebf916b 2024-11-07T17:21:21,403 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/.tmp/info/6f26fb95e1164ece9ed946608cc2e8a0 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/info/6f26fb95e1164ece9ed946608cc2e8a0 2024-11-07T17:21:21,409 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/info/6f26fb95e1164ece9ed946608cc2e8a0, entries=290, sequenceid=149, filesize=50.7 K 2024-11-07T17:21:21,410 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/.tmp/table/0af2718d91ec476387e1b8ce6ebf916b as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/table/0af2718d91ec476387e1b8ce6ebf916b 2024-11-07T17:21:21,414 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b/.tmp/testFamily/483ec65a242549b38fabbe1225deacfb 2024-11-07T17:21:21,418 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/table/0af2718d91ec476387e1b8ce6ebf916b, entries=58, sequenceid=149, filesize=9.6 K 2024-11-07T17:21:21,419 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 961ms, sequenceid=149, compaction requested=false 2024-11-07T17:21:21,420 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b/.tmp/testFamily/483ec65a242549b38fabbe1225deacfb as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b/testFamily/483ec65a242549b38fabbe1225deacfb 2024-11-07T17:21:21,423 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-07T17:21:21,424 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T17:21:21,424 INFO [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T17:21:21,424 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T17:21:21,424 DEBUG [RS_CLOSE_META-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T17:21:21,426 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b/testFamily/483ec65a242549b38fabbe1225deacfb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:21,429 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1b10e4be130aface5367bc50dee0bc3b in 437ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:21,432 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_1_-true/1b10e4be130aface5367bc50dee0bc3b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:21,433 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:21:21,433 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1b10e4be130aface5367bc50dee0bc3b: 2024-11-07T17:21:21,433 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731000036109.1b10e4be130aface5367bc50dee0bc3b. 2024-11-07T17:21:21,433 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 306dab2b507c8881dd78df8ac96ce52f, disabling compactions & flushes 2024-11-07T17:21:21,433 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:21:21,433 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:21:21,433 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. after waiting 0 ms 2024-11-07T17:21:21,433 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:21:21,433 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 306dab2b507c8881dd78df8ac96ce52f 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-07T17:21:21,449 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f/.tmp/info/1486414373fb42faa472402053000b77 is 45, key is default/info:d/1730999876989/Put/seqid=0 2024-11-07T17:21:21,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741897_1073 (size=5037) 2024-11-07T17:21:21,460 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-07T17:21:21,460 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:21,660 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:21,854 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f/.tmp/info/1486414373fb42faa472402053000b77 2024-11-07T17:21:21,861 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 306dab2b507c8881dd78df8ac96ce52f, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:21,862 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f/.tmp/info/1486414373fb42faa472402053000b77 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f/info/1486414373fb42faa472402053000b77 2024-11-07T17:21:21,868 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f/info/1486414373fb42faa472402053000b77, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T17:21:21,869 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 306dab2b507c8881dd78df8ac96ce52f in 436ms, sequenceid=6, compaction requested=false 2024-11-07T17:21:21,873 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/namespace/306dab2b507c8881dd78df8ac96ce52f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T17:21:21,874 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:21:21,874 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 306dab2b507c8881dd78df8ac96ce52f: 2024-11-07T17:21:21,874 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1730999876027.306dab2b507c8881dd78df8ac96ce52f. 2024-11-07T17:21:21,874 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d19a44f348557665f53debac39ada87f, disabling compactions & flushes 2024-11-07T17:21:21,874 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:21:21,874 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:21:21,874 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. after waiting 0 ms 2024-11-07T17:21:21,874 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:21:21,874 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d19a44f348557665f53debac39ada87f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:21,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T17:21:21,889 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f/.tmp/testFamily/0dac3ee8703f4ba1b45834965586b386 is 61, key is row-0/testFamily:testQualifier/1730999917973/Put/seqid=0 2024-11-07T17:21:21,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741898_1074 (size=5049) 2024-11-07T17:21:22,061 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:22,261 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, d19a44f348557665f53debac39ada87f, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:22,294 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f/.tmp/testFamily/0dac3ee8703f4ba1b45834965586b386 2024-11-07T17:21:22,302 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f/.tmp/testFamily/0dac3ee8703f4ba1b45834965586b386 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f/testFamily/0dac3ee8703f4ba1b45834965586b386 2024-11-07T17:21:22,308 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f/testFamily/0dac3ee8703f4ba1b45834965586b386, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:22,309 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d19a44f348557665f53debac39ada87f in 435ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:22,313 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d19a44f348557665f53debac39ada87f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:22,313 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:21:22,313 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d19a44f348557665f53debac39ada87f: 2024-11-07T17:21:22,314 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1730999916954.d19a44f348557665f53debac39ada87f. 2024-11-07T17:21:22,314 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3c6049f523e74afcad4a9e1f1737d1f2, disabling compactions & flushes 2024-11-07T17:21:22,314 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:21:22,314 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:21:22,314 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. after waiting 0 ms 2024-11-07T17:21:22,314 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:21:22,314 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3c6049f523e74afcad4a9e1f1737d1f2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:22,329 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2/.tmp/testFamily/0cd5e39d89794bc2b03038ee1708b252 is 61, key is row-0/testFamily:testQualifier/1730999971480/Put/seqid=0 2024-11-07T17:21:22,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741899_1075 (size=5049) 2024-11-07T17:21:22,462 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 25 regions to close 2024-11-07T17:21:22,462 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:22,520 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -5792281934352613241 expired clientIPAndPort=172.17.0.2:57084, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:21:22,662 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3c6049f523e74afcad4a9e1f1737d1f2, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:22,734 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2/.tmp/testFamily/0cd5e39d89794bc2b03038ee1708b252 2024-11-07T17:21:22,741 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2/.tmp/testFamily/0cd5e39d89794bc2b03038ee1708b252 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2/testFamily/0cd5e39d89794bc2b03038ee1708b252 2024-11-07T17:21:22,747 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2/testFamily/0cd5e39d89794bc2b03038ee1708b252, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:22,748 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3c6049f523e74afcad4a9e1f1737d1f2 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:22,751 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/3c6049f523e74afcad4a9e1f1737d1f2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:22,752 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:21:22,752 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3c6049f523e74afcad4a9e1f1737d1f2: 2024-11-07T17:21:22,752 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1730999968463.3c6049f523e74afcad4a9e1f1737d1f2. 2024-11-07T17:21:22,752 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 88f460756d06deb6989559d688e71bad, disabling compactions & flushes 2024-11-07T17:21:22,752 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:21:22,752 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:21:22,752 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. after waiting 0 ms 2024-11-07T17:21:22,752 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:21:22,752 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 88f460756d06deb6989559d688e71bad 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:22,768 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad/.tmp/testFamily/d989243140714b7e9d9f70861c964eae is 61, key is row-0/testFamily:testQualifier/1731000025011/Put/seqid=0 2024-11-07T17:21:22,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741900_1076 (size=5049) 2024-11-07T17:21:22,862 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:23,062 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, 88f460756d06deb6989559d688e71bad, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:23,173 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad/.tmp/testFamily/d989243140714b7e9d9f70861c964eae 2024-11-07T17:21:23,180 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad/.tmp/testFamily/d989243140714b7e9d9f70861c964eae as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad/testFamily/d989243140714b7e9d9f70861c964eae 2024-11-07T17:21:23,186 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad/testFamily/d989243140714b7e9d9f70861c964eae, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:23,187 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 88f460756d06deb6989559d688e71bad in 435ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:23,192 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_1_-true/88f460756d06deb6989559d688e71bad/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:23,192 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:21:23,192 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 88f460756d06deb6989559d688e71bad: 2024-11-07T17:21:23,193 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731000021992.88f460756d06deb6989559d688e71bad. 2024-11-07T17:21:23,193 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing dc1045097a0a1d6b49a4a2d49b1a4b05, disabling compactions & flushes 2024-11-07T17:21:23,193 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:21:23,193 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:21:23,193 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. after waiting 0 ms 2024-11-07T17:21:23,193 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:21:23,193 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing dc1045097a0a1d6b49a4a2d49b1a4b05 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:23,209 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05/.tmp/testFamily/554017c4a5f54fd79d0bb399db43b232 is 61, key is row-0/testFamily:testQualifier/1731000034568/Put/seqid=0 2024-11-07T17:21:23,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741901_1077 (size=5049) 2024-11-07T17:21:23,263 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:23,463 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 23 regions to close 2024-11-07T17:21:23,463 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, dc1045097a0a1d6b49a4a2d49b1a4b05, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:23,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T17:21:23,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T17:21:23,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T17:21:23,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T17:21:23,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-07T17:21:23,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T17:21:23,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T17:21:23,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-07T17:21:23,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T17:21:23,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T17:21:23,614 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05/.tmp/testFamily/554017c4a5f54fd79d0bb399db43b232 2024-11-07T17:21:23,621 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05/.tmp/testFamily/554017c4a5f54fd79d0bb399db43b232 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05/testFamily/554017c4a5f54fd79d0bb399db43b232 2024-11-07T17:21:23,627 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05/testFamily/554017c4a5f54fd79d0bb399db43b232, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:23,628 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dc1045097a0a1d6b49a4a2d49b1a4b05 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:23,631 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_1_-true/dc1045097a0a1d6b49a4a2d49b1a4b05/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:23,632 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:21:23,632 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for dc1045097a0a1d6b49a4a2d49b1a4b05: 2024-11-07T17:21:23,632 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731000031552.dc1045097a0a1d6b49a4a2d49b1a4b05. 2024-11-07T17:21:23,632 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ff370301e57fc94e04c045b6033bb3cb, disabling compactions & flushes 2024-11-07T17:21:23,632 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:21:23,632 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:21:23,632 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. after waiting 0 ms 2024-11-07T17:21:23,632 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:21:23,632 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ff370301e57fc94e04c045b6033bb3cb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:23,648 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb/.tmp/testFamily/3c329a3d5b68432ca7bf1b9b68ab7d7f is 61, key is row-0/testFamily:testQualifier/1730999881563/Put/seqid=0 2024-11-07T17:21:23,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741902_1078 (size=5049) 2024-11-07T17:21:23,663 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:23,863 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:24,052 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb/.tmp/testFamily/3c329a3d5b68432ca7bf1b9b68ab7d7f 2024-11-07T17:21:24,061 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb/.tmp/testFamily/3c329a3d5b68432ca7bf1b9b68ab7d7f as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb/testFamily/3c329a3d5b68432ca7bf1b9b68ab7d7f 2024-11-07T17:21:24,064 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad, ff370301e57fc94e04c045b6033bb3cb 2024-11-07T17:21:24,067 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb/testFamily/3c329a3d5b68432ca7bf1b9b68ab7d7f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:24,068 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ff370301e57fc94e04c045b6033bb3cb in 436ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:24,072 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/ff370301e57fc94e04c045b6033bb3cb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:24,073 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:21:24,073 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ff370301e57fc94e04c045b6033bb3cb: 2024-11-07T17:21:24,073 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1730999878522.ff370301e57fc94e04c045b6033bb3cb. 2024-11-07T17:21:24,073 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1df55e795a9b31b92ef935ae8ab2e252, disabling compactions & flushes 2024-11-07T17:21:24,073 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:21:24,073 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:21:24,073 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. after waiting 0 ms 2024-11-07T17:21:24,073 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:21:24,073 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1df55e795a9b31b92ef935ae8ab2e252 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:24,088 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252/.tmp/testFamily/c7c9202b5293407d99d28b75606ee036 is 61, key is row-0/testFamily:testQualifier/1730999980613/Put/seqid=0 2024-11-07T17:21:24,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741903_1079 (size=5049) 2024-11-07T17:21:24,264 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:24,464 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-07T17:21:24,464 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 1df55e795a9b31b92ef935ae8ab2e252, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:24,493 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252/.tmp/testFamily/c7c9202b5293407d99d28b75606ee036 2024-11-07T17:21:24,501 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252/.tmp/testFamily/c7c9202b5293407d99d28b75606ee036 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252/testFamily/c7c9202b5293407d99d28b75606ee036 2024-11-07T17:21:24,506 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252/testFamily/c7c9202b5293407d99d28b75606ee036, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:24,507 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1df55e795a9b31b92ef935ae8ab2e252 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:24,511 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/1df55e795a9b31b92ef935ae8ab2e252/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:24,512 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:21:24,512 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1df55e795a9b31b92ef935ae8ab2e252: 2024-11-07T17:21:24,512 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1730999977595.1df55e795a9b31b92ef935ae8ab2e252. 2024-11-07T17:21:24,512 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a64612bdb26c784d77aca9c346e0664d, disabling compactions & flushes 2024-11-07T17:21:24,512 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:21:24,512 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:21:24,512 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. after waiting 0 ms 2024-11-07T17:21:24,512 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:21:24,512 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a64612bdb26c784d77aca9c346e0664d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:24,528 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d/.tmp/testFamily/212c2771298840b59fffc2db03693bc4 is 61, key is row-0/testFamily:testQualifier/1730999905729/Put/seqid=0 2024-11-07T17:21:24,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741904_1080 (size=5049) 2024-11-07T17:21:24,664 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:24,865 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, a64612bdb26c784d77aca9c346e0664d, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:24,933 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d/.tmp/testFamily/212c2771298840b59fffc2db03693bc4 2024-11-07T17:21:24,941 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d/.tmp/testFamily/212c2771298840b59fffc2db03693bc4 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d/testFamily/212c2771298840b59fffc2db03693bc4 2024-11-07T17:21:24,948 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d/testFamily/212c2771298840b59fffc2db03693bc4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:24,949 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a64612bdb26c784d77aca9c346e0664d in 436ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:24,953 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_0_-false/a64612bdb26c784d77aca9c346e0664d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:24,953 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:21:24,953 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a64612bdb26c784d77aca9c346e0664d: 2024-11-07T17:21:24,953 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1730999902705.a64612bdb26c784d77aca9c346e0664d. 2024-11-07T17:21:24,953 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 611065383d17cc239f94ff70641327a5, disabling compactions & flushes 2024-11-07T17:21:24,954 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:21:24,954 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:21:24,954 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. after waiting 0 ms 2024-11-07T17:21:24,954 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:21:24,954 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 611065383d17cc239f94ff70641327a5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:24,969 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5/.tmp/testFamily/89d7f62cc3f7498a915442eec07bdc9a is 61, key is row-0/testFamily:testQualifier/1731000045217/Put/seqid=0 2024-11-07T17:21:24,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741905_1081 (size=5049) 2024-11-07T17:21:25,065 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:25,213 INFO [regionserver/3a0fde618c86:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-07T17:21:25,265 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 611065383d17cc239f94ff70641327a5, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:25,374 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5/.tmp/testFamily/89d7f62cc3f7498a915442eec07bdc9a 2024-11-07T17:21:25,381 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5/.tmp/testFamily/89d7f62cc3f7498a915442eec07bdc9a as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5/testFamily/89d7f62cc3f7498a915442eec07bdc9a 2024-11-07T17:21:25,387 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5/testFamily/89d7f62cc3f7498a915442eec07bdc9a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:25,388 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 611065383d17cc239f94ff70641327a5 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:25,391 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/611065383d17cc239f94ff70641327a5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:25,392 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:21:25,392 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 611065383d17cc239f94ff70641327a5: 2024-11-07T17:21:25,392 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731000042200.611065383d17cc239f94ff70641327a5. 2024-11-07T17:21:25,392 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 24ee1f613f99094716faf4222a66f5b0, disabling compactions & flushes 2024-11-07T17:21:25,392 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:21:25,392 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:21:25,392 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. after waiting 0 ms 2024-11-07T17:21:25,392 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:21:25,392 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 24ee1f613f99094716faf4222a66f5b0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:25,408 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0/.tmp/testFamily/b1b63e15362c4fadbb15d75a67db3202 is 61, key is row-0/testFamily:testQualifier/1730999962402/Put/seqid=0 2024-11-07T17:21:25,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741906_1082 (size=5049) 2024-11-07T17:21:25,465 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 18 regions to close 2024-11-07T17:21:25,466 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:25,666 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 24ee1f613f99094716faf4222a66f5b0, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:25,812 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0/.tmp/testFamily/b1b63e15362c4fadbb15d75a67db3202 2024-11-07T17:21:25,819 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0/.tmp/testFamily/b1b63e15362c4fadbb15d75a67db3202 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0/testFamily/b1b63e15362c4fadbb15d75a67db3202 2024-11-07T17:21:25,825 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0/testFamily/b1b63e15362c4fadbb15d75a67db3202, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:25,826 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 24ee1f613f99094716faf4222a66f5b0 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:25,830 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_0_-false/24ee1f613f99094716faf4222a66f5b0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:25,831 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:21:25,831 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 24ee1f613f99094716faf4222a66f5b0: 2024-11-07T17:21:25,831 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1730999959384.24ee1f613f99094716faf4222a66f5b0. 2024-11-07T17:21:25,831 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b3be295a4062dec176f6add711e118b7, disabling compactions & flushes 2024-11-07T17:21:25,831 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:21:25,831 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:21:25,831 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. after waiting 0 ms 2024-11-07T17:21:25,831 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:21:25,831 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b3be295a4062dec176f6add711e118b7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:25,846 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7/.tmp/testFamily/b3bda86d42934df88fc1458f6ad876aa is 61, key is row-0/testFamily:testQualifier/1731000018947/Put/seqid=0 2024-11-07T17:21:25,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741907_1083 (size=5049) 2024-11-07T17:21:25,866 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:26,066 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:26,251 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7/.tmp/testFamily/b3bda86d42934df88fc1458f6ad876aa 2024-11-07T17:21:26,259 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7/.tmp/testFamily/b3bda86d42934df88fc1458f6ad876aa as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7/testFamily/b3bda86d42934df88fc1458f6ad876aa 2024-11-07T17:21:26,265 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7/testFamily/b3bda86d42934df88fc1458f6ad876aa, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:26,266 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b3be295a4062dec176f6add711e118b7 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:26,266 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, b3be295a4062dec176f6add711e118b7, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:26,270 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNextAsync_1_-true/b3be295a4062dec176f6add711e118b7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:26,270 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:21:26,270 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b3be295a4062dec176f6add711e118b7: 2024-11-07T17:21:26,271 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731000015932.b3be295a4062dec176f6add711e118b7. 2024-11-07T17:21:26,271 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 272802cb3d0cc73b8a0b7d1ab03b0b71, disabling compactions & flushes 2024-11-07T17:21:26,271 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:21:26,271 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:21:26,271 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. after waiting 0 ms 2024-11-07T17:21:26,271 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:21:26,271 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 272802cb3d0cc73b8a0b7d1ab03b0b71 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:26,288 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71/.tmp/testFamily/d7940c8d08ae4f06bfb9d10c2965a2b0 is 61, key is row-0/testFamily:testQualifier/1730999878353/Put/seqid=0 2024-11-07T17:21:26,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741908_1084 (size=5049) 2024-11-07T17:21:26,467 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 16 regions to close 2024-11-07T17:21:26,467 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:26,492 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:21:26,667 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 272802cb3d0cc73b8a0b7d1ab03b0b71, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:26,693 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71/.tmp/testFamily/d7940c8d08ae4f06bfb9d10c2965a2b0 2024-11-07T17:21:26,700 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71/.tmp/testFamily/d7940c8d08ae4f06bfb9d10c2965a2b0 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71/testFamily/d7940c8d08ae4f06bfb9d10c2965a2b0 2024-11-07T17:21:26,706 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71/testFamily/d7940c8d08ae4f06bfb9d10c2965a2b0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:26,707 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 272802cb3d0cc73b8a0b7d1ab03b0b71 in 436ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:26,712 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/272802cb3d0cc73b8a0b7d1ab03b0b71/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:26,713 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:21:26,713 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 272802cb3d0cc73b8a0b7d1ab03b0b71: 2024-11-07T17:21:26,713 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1730999877268.272802cb3d0cc73b8a0b7d1ab03b0b71. 2024-11-07T17:21:26,713 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 04ecb69da5f585e0f5d77714cd8dd665, disabling compactions & flushes 2024-11-07T17:21:26,713 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:21:26,713 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:21:26,713 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. after waiting 0 ms 2024-11-07T17:21:26,713 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:21:26,713 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 04ecb69da5f585e0f5d77714cd8dd665 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:26,729 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665/.tmp/testFamily/c6de01c5adb141d58ad680c6fad0a097 is 61, key is row-0/testFamily:testQualifier/1730999936182/Put/seqid=0 2024-11-07T17:21:26,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741909_1085 (size=5049) 2024-11-07T17:21:26,867 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:27,067 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 04ecb69da5f585e0f5d77714cd8dd665, 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:27,134 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665/.tmp/testFamily/c6de01c5adb141d58ad680c6fad0a097 2024-11-07T17:21:27,141 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665/.tmp/testFamily/c6de01c5adb141d58ad680c6fad0a097 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665/testFamily/c6de01c5adb141d58ad680c6fad0a097 2024-11-07T17:21:27,146 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665/testFamily/c6de01c5adb141d58ad680c6fad0a097, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:27,147 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 04ecb69da5f585e0f5d77714cd8dd665 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:27,151 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnNext_0_-false/04ecb69da5f585e0f5d77714cd8dd665/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:27,152 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:21:27,152 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 04ecb69da5f585e0f5d77714cd8dd665: 2024-11-07T17:21:27,152 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1730999933163.04ecb69da5f585e0f5d77714cd8dd665. 2024-11-07T17:21:27,152 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c3bf6516bafcafad3967a8a32e883ac6, disabling compactions & flushes 2024-11-07T17:21:27,152 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:21:27,152 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:21:27,152 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. after waiting 0 ms 2024-11-07T17:21:27,152 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:21:27,152 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c3bf6516bafcafad3967a8a32e883ac6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:27,167 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6/.tmp/testFamily/4fa2c22528df4fe98098900045e283f9 is 61, key is row-0/testFamily:testQualifier/1731000055300/Put/seqid=0 2024-11-07T17:21:27,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741910_1086 (size=5049) 2024-11-07T17:21:27,268 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:27,468 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 14 regions to close 2024-11-07T17:21:27,468 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, c3bf6516bafcafad3967a8a32e883ac6, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:27,572 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6/.tmp/testFamily/4fa2c22528df4fe98098900045e283f9 2024-11-07T17:21:27,579 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6/.tmp/testFamily/4fa2c22528df4fe98098900045e283f9 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6/testFamily/4fa2c22528df4fe98098900045e283f9 2024-11-07T17:21:27,585 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6/testFamily/4fa2c22528df4fe98098900045e283f9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:27,586 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c3bf6516bafcafad3967a8a32e883ac6 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:27,589 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c3bf6516bafcafad3967a8a32e883ac6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:27,590 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:21:27,590 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c3bf6516bafcafad3967a8a32e883ac6: 2024-11-07T17:21:27,590 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731000052274.c3bf6516bafcafad3967a8a32e883ac6. 2024-11-07T17:21:27,590 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing aa009bbf2796b670b2b6eb293cb23b7d, disabling compactions & flushes 2024-11-07T17:21:27,590 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:21:27,590 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:21:27,590 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. after waiting 0 ms 2024-11-07T17:21:27,590 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:21:27,590 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing aa009bbf2796b670b2b6eb293cb23b7d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:27,605 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d/.tmp/testFamily/0469c0dae25b45e5a34c81c2dcc09b84 is 61, key is row-0/testFamily:testQualifier/1730999908824/Put/seqid=0 2024-11-07T17:21:27,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741911_1087 (size=5049) 2024-11-07T17:21:27,668 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:27,869 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, aa009bbf2796b670b2b6eb293cb23b7d, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:28,010 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d/.tmp/testFamily/0469c0dae25b45e5a34c81c2dcc09b84 2024-11-07T17:21:28,017 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d/.tmp/testFamily/0469c0dae25b45e5a34c81c2dcc09b84 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d/testFamily/0469c0dae25b45e5a34c81c2dcc09b84 2024-11-07T17:21:28,022 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d/testFamily/0469c0dae25b45e5a34c81c2dcc09b84, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:28,023 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aa009bbf2796b670b2b6eb293cb23b7d in 433ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:28,026 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_0_-false/aa009bbf2796b670b2b6eb293cb23b7d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:28,027 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:21:28,027 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for aa009bbf2796b670b2b6eb293cb23b7d: 2024-11-07T17:21:28,027 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1730999905801.aa009bbf2796b670b2b6eb293cb23b7d. 2024-11-07T17:21:28,027 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 72e185fd4f182be69856ff4c9c6c7377, disabling compactions & flushes 2024-11-07T17:21:28,027 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:21:28,027 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:21:28,027 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. after waiting 0 ms 2024-11-07T17:21:28,027 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:21:28,027 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 72e185fd4f182be69856ff4c9c6c7377 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:28,047 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377/.tmp/testFamily/a10f329428a443e1bd59a10799759b35 is 61, key is row-0/testFamily:testQualifier/1731000007814/Put/seqid=0 2024-11-07T17:21:28,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741912_1088 (size=5049) 2024-11-07T17:21:28,069 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:28,269 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:28,451 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377/.tmp/testFamily/a10f329428a443e1bd59a10799759b35 2024-11-07T17:21:28,459 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377/.tmp/testFamily/a10f329428a443e1bd59a10799759b35 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377/testFamily/a10f329428a443e1bd59a10799759b35 2024-11-07T17:21:28,464 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377/testFamily/a10f329428a443e1bd59a10799759b35, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:28,465 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 72e185fd4f182be69856ff4c9c6c7377 in 438ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:28,469 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnRenewLease_1_-true/72e185fd4f182be69856ff4c9c6c7377/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:28,469 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 12 regions to close 2024-11-07T17:21:28,469 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 72e185fd4f182be69856ff4c9c6c7377, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:28,469 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:21:28,469 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 72e185fd4f182be69856ff4c9c6c7377: 2024-11-07T17:21:28,469 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731000004798.72e185fd4f182be69856ff4c9c6c7377. 2024-11-07T17:21:28,470 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f87acb04cfac5bbaea483f6e8cf81cd8, disabling compactions & flushes 2024-11-07T17:21:28,470 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:28,470 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:28,470 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. after waiting 0 ms 2024-11-07T17:21:28,470 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:28,470 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f87acb04cfac5bbaea483f6e8cf81cd8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:28,486 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8/.tmp/testFamily/a7224511e31e4e5dadbdfb9429358148 is 61, key is row-0/testFamily:testQualifier/1731000065362/Put/seqid=0 2024-11-07T17:21:28,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741913_1089 (size=5049) 2024-11-07T17:21:28,669 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:28,870 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, f87acb04cfac5bbaea483f6e8cf81cd8, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:28,891 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8/.tmp/testFamily/a7224511e31e4e5dadbdfb9429358148 2024-11-07T17:21:28,897 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8/.tmp/testFamily/a7224511e31e4e5dadbdfb9429358148 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8/testFamily/a7224511e31e4e5dadbdfb9429358148 2024-11-07T17:21:28,904 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8/testFamily/a7224511e31e4e5dadbdfb9429358148, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:28,905 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f87acb04cfac5bbaea483f6e8cf81cd8 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:28,909 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNext_1_-true/f87acb04cfac5bbaea483f6e8cf81cd8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:28,910 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:28,910 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f87acb04cfac5bbaea483f6e8cf81cd8: 2024-11-07T17:21:28,910 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731000064346.f87acb04cfac5bbaea483f6e8cf81cd8. 2024-11-07T17:21:28,910 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3cfc81fcf468d7f0dd7b1f50158dd4fa, disabling compactions & flushes 2024-11-07T17:21:28,910 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:21:28,910 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:21:28,911 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. after waiting 0 ms 2024-11-07T17:21:28,911 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:21:28,911 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3cfc81fcf468d7f0dd7b1f50158dd4fa 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:28,933 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa/.tmp/testFamily/5ab8c6d1c05a461da1bbc0aa0c9c6e3d is 61, key is row-0/testFamily:testQualifier/1730999913910/Put/seqid=0 2024-11-07T17:21:28,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741914_1090 (size=5049) 2024-11-07T17:21:29,070 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:29,270 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3cfc81fcf468d7f0dd7b1f50158dd4fa, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:29,339 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa/.tmp/testFamily/5ab8c6d1c05a461da1bbc0aa0c9c6e3d 2024-11-07T17:21:29,346 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa/.tmp/testFamily/5ab8c6d1c05a461da1bbc0aa0c9c6e3d as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa/testFamily/5ab8c6d1c05a461da1bbc0aa0c9c6e3d 2024-11-07T17:21:29,351 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa/testFamily/5ab8c6d1c05a461da1bbc0aa0c9c6e3d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:29,352 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3cfc81fcf468d7f0dd7b1f50158dd4fa in 441ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:29,356 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/3cfc81fcf468d7f0dd7b1f50158dd4fa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:29,357 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:21:29,357 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3cfc81fcf468d7f0dd7b1f50158dd4fa: 2024-11-07T17:21:29,357 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1730999910888.3cfc81fcf468d7f0dd7b1f50158dd4fa. 2024-11-07T17:21:29,357 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f859399bd9a2dff5654f21dedb4a5651, disabling compactions & flushes 2024-11-07T17:21:29,357 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:21:29,357 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:21:29,357 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. after waiting 0 ms 2024-11-07T17:21:29,357 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:21:29,357 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f859399bd9a2dff5654f21dedb4a5651 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:29,373 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651/.tmp/testFamily/a24ac26e79314b12a532888fdf4ca91a is 61, key is row-0/testFamily:testQualifier/1730999893640/Put/seqid=0 2024-11-07T17:21:29,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741915_1091 (size=5049) 2024-11-07T17:21:29,470 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 9 regions to close 2024-11-07T17:21:29,471 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1603): Online Regions={f859399bd9a2dff5654f21dedb4a5651=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651., f0dc1621293458c24d1a65e07708fda5=testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5., 838a2de6cc4e0e72361bdedb53d1df08=testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08., 30c491763cb4452515bafed78f8bde08=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08., 3ebb330a2ef5c6ab0e856e72db2c5e93=hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93., 870ca7c659bec2ef0749d0a83c8f8581=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581., 63b8b46898cf84e2969939becdc3d6cb=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., 0f511d49e0cd173770e983418883dc7d=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., faf897218923774aeeb6fe4badf399ad=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.} 2024-11-07T17:21:29,471 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:29,671 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, f859399bd9a2dff5654f21dedb4a5651, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:29,777 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651/.tmp/testFamily/a24ac26e79314b12a532888fdf4ca91a 2024-11-07T17:21:29,784 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651/.tmp/testFamily/a24ac26e79314b12a532888fdf4ca91a as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651/testFamily/a24ac26e79314b12a532888fdf4ca91a 2024-11-07T17:21:29,789 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651/testFamily/a24ac26e79314b12a532888fdf4ca91a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:29,790 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f859399bd9a2dff5654f21dedb4a5651 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:29,794 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/f859399bd9a2dff5654f21dedb4a5651/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:29,794 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:21:29,794 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f859399bd9a2dff5654f21dedb4a5651: 2024-11-07T17:21:29,794 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1730999890614.f859399bd9a2dff5654f21dedb4a5651. 2024-11-07T17:21:29,794 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f0dc1621293458c24d1a65e07708fda5, disabling compactions & flushes 2024-11-07T17:21:29,794 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:21:29,795 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:21:29,795 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. after waiting 0 ms 2024-11-07T17:21:29,795 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:21:29,795 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f0dc1621293458c24d1a65e07708fda5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:29,815 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5/.tmp/testFamily/02fe20aadca548f4aa527080b6d385f8 is 61, key is row-0/testFamily:testQualifier/1730999931619/Put/seqid=0 2024-11-07T17:21:29,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741916_1092 (size=5049) 2024-11-07T17:21:29,871 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:30,071 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, f0dc1621293458c24d1a65e07708fda5, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:30,219 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5/.tmp/testFamily/02fe20aadca548f4aa527080b6d385f8 2024-11-07T17:21:30,226 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5/.tmp/testFamily/02fe20aadca548f4aa527080b6d385f8 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5/testFamily/02fe20aadca548f4aa527080b6d385f8 2024-11-07T17:21:30,232 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5/testFamily/02fe20aadca548f4aa527080b6d385f8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:30,233 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f0dc1621293458c24d1a65e07708fda5 in 438ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:30,236 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnClose_0_-false/f0dc1621293458c24d1a65e07708fda5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:30,237 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:21:30,237 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f0dc1621293458c24d1a65e07708fda5: 2024-11-07T17:21:30,237 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1730999928599.f0dc1621293458c24d1a65e07708fda5. 2024-11-07T17:21:30,237 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 838a2de6cc4e0e72361bdedb53d1df08, disabling compactions & flushes 2024-11-07T17:21:30,237 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:30,237 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:30,237 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. after waiting 0 ms 2024-11-07T17:21:30,237 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:30,237 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 838a2de6cc4e0e72361bdedb53d1df08 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:30,253 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08/.tmp/testFamily/dac87520327a498d867b06e5e57c9aba is 61, key is row-0/testFamily:testQualifier/1731000004742/Put/seqid=0 2024-11-07T17:21:30,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741917_1093 (size=5049) 2024-11-07T17:21:30,271 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:30,472 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 7 regions to close 2024-11-07T17:21:30,472 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1603): Online Regions={838a2de6cc4e0e72361bdedb53d1df08=testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08., 30c491763cb4452515bafed78f8bde08=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08., 3ebb330a2ef5c6ab0e856e72db2c5e93=hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93., 870ca7c659bec2ef0749d0a83c8f8581=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581., 63b8b46898cf84e2969939becdc3d6cb=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., 0f511d49e0cd173770e983418883dc7d=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., faf897218923774aeeb6fe4badf399ad=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.} 2024-11-07T17:21:30,472 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:30,658 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08/.tmp/testFamily/dac87520327a498d867b06e5e57c9aba 2024-11-07T17:21:30,664 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08/.tmp/testFamily/dac87520327a498d867b06e5e57c9aba as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08/testFamily/dac87520327a498d867b06e5e57c9aba 2024-11-07T17:21:30,670 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08/testFamily/dac87520327a498d867b06e5e57c9aba, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:30,671 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 838a2de6cc4e0e72361bdedb53d1df08 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:30,672 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 838a2de6cc4e0e72361bdedb53d1df08, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:30,674 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextException_1_-true/838a2de6cc4e0e72361bdedb53d1df08/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:30,675 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:30,675 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 838a2de6cc4e0e72361bdedb53d1df08: 2024-11-07T17:21:30,675 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731000001724.838a2de6cc4e0e72361bdedb53d1df08. 2024-11-07T17:21:30,675 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 30c491763cb4452515bafed78f8bde08, disabling compactions & flushes 2024-11-07T17:21:30,675 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:21:30,675 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:21:30,675 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. after waiting 0 ms 2024-11-07T17:21:30,675 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:21:30,675 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 30c491763cb4452515bafed78f8bde08 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:30,691 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08/.tmp/testFamily/60ed3cc75af44b3fbd03c06f2d8df516 is 61, key is row-0/testFamily:testQualifier/1730999977540/Put/seqid=0 2024-11-07T17:21:30,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741918_1094 (size=5049) 2024-11-07T17:21:30,872 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:31,072 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 30c491763cb4452515bafed78f8bde08, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:31,095 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08/.tmp/testFamily/60ed3cc75af44b3fbd03c06f2d8df516 2024-11-07T17:21:31,102 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08/.tmp/testFamily/60ed3cc75af44b3fbd03c06f2d8df516 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08/testFamily/60ed3cc75af44b3fbd03c06f2d8df516 2024-11-07T17:21:31,107 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08/testFamily/60ed3cc75af44b3fbd03c06f2d8df516, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:31,108 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 30c491763cb4452515bafed78f8bde08 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:31,112 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/30c491763cb4452515bafed78f8bde08/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:31,113 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:21:31,113 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 30c491763cb4452515bafed78f8bde08: 2024-11-07T17:21:31,113 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1730999974523.30c491763cb4452515bafed78f8bde08. 2024-11-07T17:21:31,113 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3ebb330a2ef5c6ab0e856e72db2c5e93, disabling compactions & flushes 2024-11-07T17:21:31,113 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:21:31,113 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:21:31,113 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. after waiting 0 ms 2024-11-07T17:21:31,113 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:21:31,113 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3ebb330a2ef5c6ab0e856e72db2c5e93 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:31,129 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93/.tmp/testFamily/4eb51615d0f6466a886318ed73bee4b6 is 61, key is row-0/testFamily:testQualifier/1730999922056/Put/seqid=0 2024-11-07T17:21:31,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741919_1095 (size=5049) 2024-11-07T17:21:31,273 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:31,473 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 5 regions to close 2024-11-07T17:21:31,473 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1603): Online Regions={3ebb330a2ef5c6ab0e856e72db2c5e93=hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93., 870ca7c659bec2ef0749d0a83c8f8581=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581., 63b8b46898cf84e2969939becdc3d6cb=testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb., 0f511d49e0cd173770e983418883dc7d=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., faf897218923774aeeb6fe4badf399ad=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.} 2024-11-07T17:21:31,473 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 3ebb330a2ef5c6ab0e856e72db2c5e93, 63b8b46898cf84e2969939becdc3d6cb, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:31,533 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93/.tmp/testFamily/4eb51615d0f6466a886318ed73bee4b6 2024-11-07T17:21:31,540 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93/.tmp/testFamily/4eb51615d0f6466a886318ed73bee4b6 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93/testFamily/4eb51615d0f6466a886318ed73bee4b6 2024-11-07T17:21:31,545 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93/testFamily/4eb51615d0f6466a886318ed73bee4b6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:31,546 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3ebb330a2ef5c6ab0e856e72db2c5e93 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:31,549 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnClose_0_-false/3ebb330a2ef5c6ab0e856e72db2c5e93/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:31,550 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:21:31,550 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3ebb330a2ef5c6ab0e856e72db2c5e93: 2024-11-07T17:21:31,550 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1730999921037.3ebb330a2ef5c6ab0e856e72db2c5e93. 2024-11-07T17:21:31,550 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 870ca7c659bec2ef0749d0a83c8f8581, disabling compactions & flushes 2024-11-07T17:21:31,550 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:21:31,550 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:21:31,550 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. after waiting 0 ms 2024-11-07T17:21:31,550 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:21:31,550 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 870ca7c659bec2ef0749d0a83c8f8581 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:31,565 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581/.tmp/testFamily/6dc570e53a484af5896f742d44a6fcfc is 61, key is row-0/testFamily:testQualifier/1730999940262/Put/seqid=0 2024-11-07T17:21:31,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741920_1096 (size=5049) 2024-11-07T17:21:31,673 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 63b8b46898cf84e2969939becdc3d6cb, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:31,772 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T17:21:31,873 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 63b8b46898cf84e2969939becdc3d6cb, 870ca7c659bec2ef0749d0a83c8f8581, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:31,970 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581/.tmp/testFamily/6dc570e53a484af5896f742d44a6fcfc 2024-11-07T17:21:31,977 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581/.tmp/testFamily/6dc570e53a484af5896f742d44a6fcfc as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581/testFamily/6dc570e53a484af5896f742d44a6fcfc 2024-11-07T17:21:31,982 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581/testFamily/6dc570e53a484af5896f742d44a6fcfc, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:31,983 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 870ca7c659bec2ef0749d0a83c8f8581 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:31,987 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/870ca7c659bec2ef0749d0a83c8f8581/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:31,987 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:21:31,988 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 870ca7c659bec2ef0749d0a83c8f8581: 2024-11-07T17:21:31,988 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1730999937242.870ca7c659bec2ef0749d0a83c8f8581. 2024-11-07T17:21:31,988 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 63b8b46898cf84e2969939becdc3d6cb, disabling compactions & flushes 2024-11-07T17:21:31,988 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:21:31,988 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:21:31,988 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. after waiting 0 ms 2024-11-07T17:21:31,988 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:21:31,988 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 63b8b46898cf84e2969939becdc3d6cb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:32,003 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb/.tmp/testFamily/c8c5c93287834650938e72527a485572 is 61, key is row-0/testFamily:testQualifier/1731000012887/Put/seqid=0 2024-11-07T17:21:32,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741921_1097 (size=5049) 2024-11-07T17:21:32,074 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 63b8b46898cf84e2969939becdc3d6cb, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:32,274 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, 63b8b46898cf84e2969939becdc3d6cb, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:32,408 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb/.tmp/testFamily/c8c5c93287834650938e72527a485572 2024-11-07T17:21:32,415 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb/.tmp/testFamily/c8c5c93287834650938e72527a485572 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb/testFamily/c8c5c93287834650938e72527a485572 2024-11-07T17:21:32,420 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb/testFamily/c8c5c93287834650938e72527a485572, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:32,421 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 63b8b46898cf84e2969939becdc3d6cb in 433ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:32,425 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/63b8b46898cf84e2969939becdc3d6cb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:32,426 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:21:32,426 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 63b8b46898cf84e2969939becdc3d6cb: 2024-11-07T17:21:32,426 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731000009871.63b8b46898cf84e2969939becdc3d6cb. 2024-11-07T17:21:32,426 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0f511d49e0cd173770e983418883dc7d, disabling compactions & flushes 2024-11-07T17:21:32,426 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:32,426 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:32,426 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. after waiting 0 ms 2024-11-07T17:21:32,426 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:32,426 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0f511d49e0cd173770e983418883dc7d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:32,442 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d/.tmp/testFamily/37ddfdd6e58848639a3532570c09cbdf is 61, key is row-0/testFamily:testQualifier/1731000077430/Put/seqid=0 2024-11-07T17:21:32,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741922_1098 (size=5049) 2024-11-07T17:21:32,474 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-07T17:21:32,474 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1603): Online Regions={0f511d49e0cd173770e983418883dc7d=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d., faf897218923774aeeb6fe4badf399ad=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad.} 2024-11-07T17:21:32,474 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:32,674 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on 0f511d49e0cd173770e983418883dc7d, faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:32,847 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d/.tmp/testFamily/37ddfdd6e58848639a3532570c09cbdf 2024-11-07T17:21:32,854 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d/.tmp/testFamily/37ddfdd6e58848639a3532570c09cbdf as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d/testFamily/37ddfdd6e58848639a3532570c09cbdf 2024-11-07T17:21:32,859 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d/testFamily/37ddfdd6e58848639a3532570c09cbdf, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:32,860 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0f511d49e0cd173770e983418883dc7d in 434ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:32,864 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0f511d49e0cd173770e983418883dc7d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:32,865 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:32,865 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0f511d49e0cd173770e983418883dc7d: 2024-11-07T17:21:32,865 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731000074413.0f511d49e0cd173770e983418883dc7d. 2024-11-07T17:21:32,865 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing faf897218923774aeeb6fe4badf399ad, disabling compactions & flushes 2024-11-07T17:21:32,865 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:21:32,865 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:21:32,865 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. after waiting 0 ms 2024-11-07T17:21:32,865 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:21:32,865 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing faf897218923774aeeb6fe4badf399ad 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T17:21:32,874 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:32,881 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad/.tmp/testFamily/c8c7de15868342bbaf460a4ccb418c65 is 61, key is row-0/testFamily:testQualifier/1730999992671/Put/seqid=0 2024-11-07T17:21:32,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741923_1099 (size=5049) 2024-11-07T17:21:33,075 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:33,275 DEBUG [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1629): Waiting on faf897218923774aeeb6fe4badf399ad 2024-11-07T17:21:33,285 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad/.tmp/testFamily/c8c7de15868342bbaf460a4ccb418c65 2024-11-07T17:21:33,291 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad/.tmp/testFamily/c8c7de15868342bbaf460a4ccb418c65 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad/testFamily/c8c7de15868342bbaf460a4ccb418c65 2024-11-07T17:21:33,297 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad/testFamily/c8c7de15868342bbaf460a4ccb418c65, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T17:21:33,298 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for faf897218923774aeeb6fe4badf399ad in 433ms, sequenceid=8, compaction requested=false 2024-11-07T17:21:33,302 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/faf897218923774aeeb6fe4badf399ad/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T17:21:33,303 INFO [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:21:33,303 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for faf897218923774aeeb6fe4badf399ad: 2024-11-07T17:21:33,303 DEBUG [RS_CLOSE_REGION-regionserver/3a0fde618c86:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1730999989654.faf897218923774aeeb6fe4badf399ad. 2024-11-07T17:21:33,475 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1250): stopping server 3a0fde618c86,38099,1730999843825; all regions closed. 2024-11-07T17:21:33,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741834_1010 (size=56731) 2024-11-07T17:21:33,484 DEBUG [RS:0;3a0fde618c86:38099 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/oldWALs 2024-11-07T17:21:33,484 INFO [RS:0;3a0fde618c86:38099 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 3a0fde618c86%2C38099%2C1730999843825.meta:.meta(num 1730999875678) 2024-11-07T17:21:33,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741833_1009 (size=60562) 2024-11-07T17:21:33,488 DEBUG [RS:0;3a0fde618c86:38099 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/oldWALs 2024-11-07T17:21:33,489 INFO [RS:0;3a0fde618c86:38099 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 3a0fde618c86%2C38099%2C1730999843825:(num 1730999875203) 2024-11-07T17:21:33,489 DEBUG [RS:0;3a0fde618c86:38099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:33,489 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T17:21:33,489 INFO [RS:0;3a0fde618c86:38099 {}] hbase.ChoreService(370): Chore service for: regionserver/3a0fde618c86:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-11-07T17:21:33,489 INFO [regionserver/3a0fde618c86:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T17:21:33,490 INFO [RS:0;3a0fde618c86:38099 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38099 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-07T17:21:33,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T17:21:33,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T17:21:33,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-07T17:21:33,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T17:21:33,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3a0fde618c86,38099,1730999843825 2024-11-07T17:21:33,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T17:21:33,494 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$357/0x00007f196c8d4e10@5885830b rejected from java.util.concurrent.ThreadPoolExecutor@1d87167f[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 15] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-07T17:21:33,496 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3a0fde618c86,38099,1730999843825] 2024-11-07T17:21:33,496 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 3a0fde618c86,38099,1730999843825; numProcessing=1 2024-11-07T17:21:33,497 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/3a0fde618c86,38099,1730999843825 already deleted, retry=false 2024-11-07T17:21:33,497 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 3a0fde618c86,38099,1730999843825 expired; onlineServers=0 2024-11-07T17:21:33,497 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '3a0fde618c86,36799,1730999842914' ***** 2024-11-07T17:21:33,497 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T17:21:33,497 DEBUG [M:0;3a0fde618c86:36799 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3160cd8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3a0fde618c86/172.17.0.2:0 2024-11-07T17:21:33,497 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HRegionServer(1224): stopping server 3a0fde618c86,36799,1730999842914 2024-11-07T17:21:33,498 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HRegionServer(1250): stopping server 3a0fde618c86,36799,1730999842914; all regions closed. 2024-11-07T17:21:33,498 DEBUG [M:0;3a0fde618c86:36799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T17:21:33,498 DEBUG [M:0;3a0fde618c86:36799 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T17:21:33,498 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T17:21:33,498 DEBUG [M:0;3a0fde618c86:36799 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T17:21:33,498 DEBUG [master/3a0fde618c86:0:becomeActiveMaster-HFileCleaner.small.0-1730999845020 {}] cleaner.HFileCleaner(306): Exit Thread[master/3a0fde618c86:0:becomeActiveMaster-HFileCleaner.small.0-1730999845020,5,FailOnTimeoutGroup] 2024-11-07T17:21:33,498 DEBUG [master/3a0fde618c86:0:becomeActiveMaster-HFileCleaner.large.0-1730999845020 {}] cleaner.HFileCleaner(306): Exit Thread[master/3a0fde618c86:0:becomeActiveMaster-HFileCleaner.large.0-1730999845020,5,FailOnTimeoutGroup] 2024-11-07T17:21:33,498 INFO [M:0;3a0fde618c86:36799 {}] hbase.ChoreService(370): Chore service for: master/3a0fde618c86:0 had [] on shutdown 2024-11-07T17:21:33,498 DEBUG [M:0;3a0fde618c86:36799 {}] master.HMaster(1733): Stopping service threads 2024-11-07T17:21:33,498 INFO [M:0;3a0fde618c86:36799 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T17:21:33,499 ERROR [M:0;3a0fde618c86:36799 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] 2024-11-07T17:21:33,499 INFO [M:0;3a0fde618c86:36799 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T17:21:33,499 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T17:21:33,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T17:21:33,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T17:21:33,500 DEBUG [M:0;3a0fde618c86:36799 {}] zookeeper.ZKUtil(347): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T17:21:33,500 WARN [M:0;3a0fde618c86:36799 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T17:21:33,500 INFO [M:0;3a0fde618c86:36799 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-07T17:21:33,500 INFO [M:0;3a0fde618c86:36799 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T17:21:33,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T17:21:33,500 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T17:21:33,500 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:21:33,500 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:21:33,500 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T17:21:33,500 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:21:33,501 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.24 KB heapSize=519.69 KB 2024-11-07T17:21:33,518 DEBUG [M:0;3a0fde618c86:36799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ccd786b468a4119acc6fc24d084bc14 is 82, key is hbase:meta,,1/info:regioninfo/1730999875901/Put/seqid=0 2024-11-07T17:21:33,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741924_1100 (size=5672) 2024-11-07T17:21:33,529 INFO [regionserver/3a0fde618c86:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T17:21:33,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T17:21:33,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38099-0x10183bcea110001, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T17:21:33,596 INFO [RS:0;3a0fde618c86:38099 {}] regionserver.HRegionServer(1307): Exiting; stopping=3a0fde618c86,38099,1730999843825; zookeeper connection closed. 2024-11-07T17:21:33,596 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5be7a3d6 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5be7a3d6 2024-11-07T17:21:33,596 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T17:21:33,923 INFO [M:0;3a0fde618c86:36799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ccd786b468a4119acc6fc24d084bc14 2024-11-07T17:21:33,947 DEBUG [M:0;3a0fde618c86:36799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3b42522b42024187825af5d2a44a8a47 is 830, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1730999877787/Put/seqid=0 2024-11-07T17:21:33,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741925_1101 (size=29496) 2024-11-07T17:21:34,352 INFO [M:0;3a0fde618c86:36799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=447.69 KB at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3b42522b42024187825af5d2a44a8a47 2024-11-07T17:21:34,355 ERROR [LeaseRenewer:jenkins@localhost:34871 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:34871,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T17:21:34,358 INFO [M:0;3a0fde618c86:36799 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b42522b42024187825af5d2a44a8a47 2024-11-07T17:21:34,373 DEBUG [M:0;3a0fde618c86:36799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/205e3ef95dd44b8fac374fc45dfa2dd1 is 69, key is 3a0fde618c86,38099,1730999843825/rs:state/1730999845040/Put/seqid=0 2024-11-07T17:21:34,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741926_1102 (size=5156) 2024-11-07T17:21:34,778 INFO [M:0;3a0fde618c86:36799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/205e3ef95dd44b8fac374fc45dfa2dd1 2024-11-07T17:21:34,783 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ccd786b468a4119acc6fc24d084bc14 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8ccd786b468a4119acc6fc24d084bc14 2024-11-07T17:21:34,788 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8ccd786b468a4119acc6fc24d084bc14, entries=8, sequenceid=830, filesize=5.5 K 2024-11-07T17:21:34,789 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3b42522b42024187825af5d2a44a8a47 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3b42522b42024187825af5d2a44a8a47 2024-11-07T17:21:34,793 INFO [M:0;3a0fde618c86:36799 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b42522b42024187825af5d2a44a8a47 2024-11-07T17:21:34,794 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3b42522b42024187825af5d2a44a8a47, entries=92, sequenceid=830, filesize=28.8 K 2024-11-07T17:21:34,794 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/205e3ef95dd44b8fac374fc45dfa2dd1 as hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/205e3ef95dd44b8fac374fc45dfa2dd1 2024-11-07T17:21:34,799 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34871/user/jenkins/test-data/cba5afe6-5403-cf5a-669f-b4f6c1293392/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/205e3ef95dd44b8fac374fc45dfa2dd1, entries=1, sequenceid=830, filesize=5.0 K 2024-11-07T17:21:34,800 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.24 KB/459001, heapSize ~519.39 KB/531856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1300ms, sequenceid=830, compaction requested=false 2024-11-07T17:21:34,801 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T17:21:34,801 DEBUG [M:0;3a0fde618c86:36799 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T17:21:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741830_1006 (size=513870) 2024-11-07T17:21:34,804 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T17:21:34,804 INFO [M:0;3a0fde618c86:36799 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-07T17:21:34,804 INFO [M:0;3a0fde618c86:36799 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36799 2024-11-07T17:21:34,806 DEBUG [M:0;3a0fde618c86:36799 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/3a0fde618c86,36799,1730999842914 already deleted, retry=false 2024-11-07T17:21:34,908 INFO [M:0;3a0fde618c86:36799 {}] regionserver.HRegionServer(1307): Exiting; stopping=3a0fde618c86,36799,1730999842914; zookeeper connection closed. 2024-11-07T17:21:34,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T17:21:34,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36799-0x10183bcea110000, quorum=127.0.0.1:57710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T17:21:34,914 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a0f56e0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T17:21:34,916 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6dd4310{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T17:21:34,916 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T17:21:34,916 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@da2227{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T17:21:34,916 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@534ddf03{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/hadoop.log.dir/,STOPPED} 2024-11-07T17:21:34,919 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T17:21:34,919 WARN [BP-1071349954-172.17.0.2-1730999839187 heartbeating to localhost/127.0.0.1:34871 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T17:21:34,919 WARN [BP-1071349954-172.17.0.2-1730999839187 heartbeating to localhost/127.0.0.1:34871 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1071349954-172.17.0.2-1730999839187 (Datanode Uuid 5710eab7-4319-48ee-8067-f4b9cc2baa08) service to localhost/127.0.0.1:34871 2024-11-07T17:21:34,919 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T17:21:34,920 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1/dfs/data/data1/current/BP-1071349954-172.17.0.2-1730999839187 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T17:21:34,920 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/cluster_56e5292e-10aa-c541-b4dd-210651ddd6b1/dfs/data/data2/current/BP-1071349954-172.17.0.2-1730999839187 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T17:21:34,921 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T17:21:34,927 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T17:21:34,928 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T17:21:34,928 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T17:21:34,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T17:21:34,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/0b0b7dc1-6279-2b7d-f286-9c48e8f73e34/hadoop.log.dir/,STOPPED} 2024-11-07T17:21:34,938 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-07T17:21:34,979 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down