2024-11-08 06:17:14,187 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 06:17:14,199 main DEBUG Took 0.010207 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 06:17:14,199 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 06:17:14,199 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 06:17:14,200 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 06:17:14,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,220 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 06:17:14,237 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,239 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,240 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,240 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,242 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,243 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,245 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,246 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,247 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,248 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,248 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,249 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,249 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,250 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,250 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,252 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,252 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 06:17:14,253 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,253 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 06:17:14,255 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 06:17:14,257 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 06:17:14,259 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 06:17:14,259 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 06:17:14,261 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 06:17:14,261 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 06:17:14,273 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 06:17:14,276 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 06:17:14,278 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 06:17:14,279 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 06:17:14,279 main DEBUG createAppenders(={Console}) 2024-11-08 06:17:14,280 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-08 06:17:14,281 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-08 06:17:14,281 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-08 06:17:14,282 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 06:17:14,282 main DEBUG OutputStream closed 2024-11-08 06:17:14,283 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 06:17:14,283 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 06:17:14,283 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-08 06:17:14,368 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 06:17:14,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 06:17:14,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 06:17:14,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 06:17:14,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 06:17:14,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 06:17:14,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 06:17:14,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 06:17:14,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 06:17:14,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 06:17:14,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 06:17:14,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 06:17:14,378 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 06:17:14,378 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 06:17:14,378 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 06:17:14,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 06:17:14,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 06:17:14,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 06:17:14,383 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 06:17:14,384 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-08 06:17:14,384 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 06:17:14,385 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-08T06:17:14,610 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-08 06:17:14,612 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 06:17:14,613 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T06:17:14,682 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118 2024-11-08T06:17:14,714 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T06:17:14,717 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091, deleteOnExit=true 2024-11-08T06:17:14,718 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-08T06:17:14,719 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/test.cache.data in system properties and HBase conf 2024-11-08T06:17:14,719 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T06:17:14,720 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/hadoop.log.dir in system properties and HBase conf 2024-11-08T06:17:14,721 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T06:17:14,721 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T06:17:14,722 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-08T06:17:14,819 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T06:17:14,905 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T06:17:14,908 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T06:17:14,909 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T06:17:14,909 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T06:17:14,910 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T06:17:14,910 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T06:17:14,911 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T06:17:14,911 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T06:17:14,911 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T06:17:14,912 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T06:17:14,912 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/nfs.dump.dir in system properties and HBase conf 2024-11-08T06:17:14,913 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/java.io.tmpdir in system properties and HBase conf 2024-11-08T06:17:14,913 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T06:17:14,913 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T06:17:14,914 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T06:17:15,704 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T06:17:15,778 INFO [Time-limited test {}] log.Log(170): Logging initialized @2290ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T06:17:15,852 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T06:17:15,913 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T06:17:15,933 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T06:17:15,933 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T06:17:15,934 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T06:17:15,946 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T06:17:15,949 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/hadoop.log.dir/,AVAILABLE} 2024-11-08T06:17:15,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T06:17:16,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/java.io.tmpdir/jetty-localhost-44521-hadoop-hdfs-3_4_1-tests_jar-_-any-13969879330966069158/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T06:17:16,172 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:44521} 2024-11-08T06:17:16,172 INFO [Time-limited test {}] server.Server(415): Started @2686ms 2024-11-08T06:17:16,558 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T06:17:16,565 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T06:17:16,566 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T06:17:16,566 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T06:17:16,566 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T06:17:16,567 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/hadoop.log.dir/,AVAILABLE} 2024-11-08T06:17:16,568 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T06:17:16,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@183020ad{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/java.io.tmpdir/jetty-localhost-38877-hadoop-hdfs-3_4_1-tests_jar-_-any-9947855895908154764/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T06:17:16,693 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:38877} 2024-11-08T06:17:16,693 INFO [Time-limited test {}] server.Server(415): Started @3207ms 2024-11-08T06:17:16,748 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T06:17:17,226 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091/dfs/data/data2/current/BP-1513049708-172.17.0.2-1731046635478/current, will proceed with Du for space computation calculation, 2024-11-08T06:17:17,226 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091/dfs/data/data1/current/BP-1513049708-172.17.0.2-1731046635478/current, will proceed with Du for space computation calculation, 2024-11-08T06:17:17,261 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T06:17:17,311 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda9bea4b00c908b0 with lease ID 0xc16e283ad3141ea: Processing first storage report for DS-502572dd-0ae8-4038-9112-bab8f9c64d4d from datanode DatanodeRegistration(127.0.0.1:35001, datanodeUuid=a7a094ce-9504-4741-ae98-5162a1401787, infoPort=34773, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=668393702;c=1731046635478) 2024-11-08T06:17:17,313 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda9bea4b00c908b0 with lease ID 0xc16e283ad3141ea: from storage DS-502572dd-0ae8-4038-9112-bab8f9c64d4d node DatanodeRegistration(127.0.0.1:35001, datanodeUuid=a7a094ce-9504-4741-ae98-5162a1401787, infoPort=34773, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=668393702;c=1731046635478), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T06:17:17,313 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda9bea4b00c908b0 with lease ID 0xc16e283ad3141ea: Processing first storage report for DS-9f64c026-109a-41c9-8c54-50cbe433c802 from datanode DatanodeRegistration(127.0.0.1:35001, datanodeUuid=a7a094ce-9504-4741-ae98-5162a1401787, infoPort=34773, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=668393702;c=1731046635478) 2024-11-08T06:17:17,313 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda9bea4b00c908b0 with lease ID 0xc16e283ad3141ea: from storage DS-9f64c026-109a-41c9-8c54-50cbe433c802 node DatanodeRegistration(127.0.0.1:35001, datanodeUuid=a7a094ce-9504-4741-ae98-5162a1401787, infoPort=34773, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=668393702;c=1731046635478), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T06:17:17,405 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118 2024-11-08T06:17:17,477 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091/zookeeper_0, clientPort=49997, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T06:17:17,488 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=49997 2024-11-08T06:17:17,498 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T06:17:17,500 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T06:17:17,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741825_1001 (size=7) 2024-11-08T06:17:18,161 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a with version=8 2024-11-08T06:17:18,161 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/hbase-staging 2024-11-08T06:17:18,293 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T06:17:18,563 INFO [Time-limited test {}] client.ConnectionUtils(129): master/d1f4bb6aa14f:0 server-side Connection retries=9 2024-11-08T06:17:18,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T06:17:18,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T06:17:18,584 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T06:17:18,584 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T06:17:18,584 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T06:17:18,715 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T06:17:18,774 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T06:17:18,783 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T06:17:18,787 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T06:17:18,813 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 13148 (auto-detected) 2024-11-08T06:17:18,814 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T06:17:18,832 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46003 2024-11-08T06:17:18,840 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T06:17:18,842 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T06:17:18,854 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:46003 connecting to ZooKeeper ensemble=127.0.0.1:49997 2024-11-08T06:17:18,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:460030x0, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T06:17:18,887 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46003-0x1018686f4fc0000 connected 2024-11-08T06:17:18,917 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T06:17:18,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T06:17:18,923 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T06:17:18,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46003 2024-11-08T06:17:18,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46003 2024-11-08T06:17:18,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46003 2024-11-08T06:17:18,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46003 2024-11-08T06:17:18,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46003 2024-11-08T06:17:18,935 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a, hbase.cluster.distributed=false 2024-11-08T06:17:18,999 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/d1f4bb6aa14f:0 server-side Connection retries=9 2024-11-08T06:17:19,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T06:17:19,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T06:17:19,000 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T06:17:19,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T06:17:19,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T06:17:19,002 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T06:17:19,005 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T06:17:19,005 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36803 2024-11-08T06:17:19,007 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T06:17:19,012 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T06:17:19,013 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T06:17:19,016 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T06:17:19,019 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36803 connecting to ZooKeeper ensemble=127.0.0.1:49997 2024-11-08T06:17:19,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368030x0, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T06:17:19,023 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36803-0x1018686f4fc0001 connected 2024-11-08T06:17:19,024 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T06:17:19,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T06:17:19,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T06:17:19,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36803 2024-11-08T06:17:19,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36803 2024-11-08T06:17:19,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36803 2024-11-08T06:17:19,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36803 2024-11-08T06:17:19,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36803 2024-11-08T06:17:19,033 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:17:19,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T06:17:19,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T06:17:19,043 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:17:19,048 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d1f4bb6aa14f:46003 2024-11-08T06:17:19,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T06:17:19,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T06:17:19,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:19,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:19,066 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T06:17:19,067 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T06:17:19,067 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d1f4bb6aa14f,46003,1731046638286 from backup master directory 2024-11-08T06:17:19,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:17:19,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T06:17:19,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T06:17:19,071 WARN [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T06:17:19,071 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:17:19,074 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T06:17:19,076 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T06:17:19,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741826_1002 (size=42) 2024-11-08T06:17:19,549 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/hbase.id with ID: 9f5bf3b5-d672-4cfd-b7a8-e7a14c1c6d2c 2024-11-08T06:17:19,591 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T06:17:19,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:19,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:19,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741827_1003 (size=196) 2024-11-08T06:17:20,047 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:17:20,049 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T06:17:20,066 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:17:20,070 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T06:17:20,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741828_1004 (size=1189) 2024-11-08T06:17:20,531 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store 2024-11-08T06:17:20,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741829_1005 (size=34) 2024-11-08T06:17:20,613 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T06:17:20,614 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:17:20,615 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T06:17:20,615 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:17:20,616 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:17:20,616 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T06:17:20,616 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:17:20,616 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:17:20,617 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T06:17:20,626 WARN [master/d1f4bb6aa14f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/.initializing 2024-11-08T06:17:20,626 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/WALs/d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:17:20,636 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T06:17:20,658 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1f4bb6aa14f%2C46003%2C1731046638286, suffix=, logDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/WALs/d1f4bb6aa14f,46003,1731046638286, archiveDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/oldWALs, maxLogs=10 2024-11-08T06:17:20,693 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/WALs/d1f4bb6aa14f,46003,1731046638286/d1f4bb6aa14f%2C46003%2C1731046638286.1731046640666, exclude list is [], retry=0 2024-11-08T06:17:20,717 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35001,DS-502572dd-0ae8-4038-9112-bab8f9c64d4d,DISK] 2024-11-08T06:17:20,721 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T06:17:20,766 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/WALs/d1f4bb6aa14f,46003,1731046638286/d1f4bb6aa14f%2C46003%2C1731046638286.1731046640666 2024-11-08T06:17:20,767 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34773:34773)] 2024-11-08T06:17:20,767 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:17:20,768 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:17:20,772 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,773 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T06:17:20,859 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:20,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T06:17:20,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T06:17:20,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:20,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:17:20,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T06:17:20,876 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:20,877 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:17:20,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T06:17:20,881 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:20,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:17:20,886 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,888 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,898 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T06:17:20,903 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T06:17:20,910 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:17:20,912 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60245307, jitterRate=-0.10227496922016144}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T06:17:20,918 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T06:17:20,919 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T06:17:20,953 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9392ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:17:20,995 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-08T06:17:21,011 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T06:17:21,011 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T06:17:21,014 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T06:17:21,016 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T06:17:21,023 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 6 msec 2024-11-08T06:17:21,023 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T06:17:21,056 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T06:17:21,075 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T06:17:21,085 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-08T06:17:21,088 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T06:17:21,089 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T06:17:21,091 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-08T06:17:21,094 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T06:17:21,101 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T06:17:21,106 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-08T06:17:21,107 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T06:17:21,112 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T06:17:21,129 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T06:17:21,131 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T06:17:21,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T06:17:21,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T06:17:21,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:21,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:21,137 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=d1f4bb6aa14f,46003,1731046638286, sessionid=0x1018686f4fc0000, setting cluster-up flag (Was=false) 2024-11-08T06:17:21,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:21,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:21,161 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T06:17:21,163 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:17:21,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:21,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:17:21,176 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T06:17:21,179 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:17:21,256 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d1f4bb6aa14f:36803 2024-11-08T06:17:21,259 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1008): ClusterId : 9f5bf3b5-d672-4cfd-b7a8-e7a14c1c6d2c 2024-11-08T06:17:21,263 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T06:17:21,269 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T06:17:21,269 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T06:17:21,276 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T06:17:21,276 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a7a6c16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:17:21,280 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1f4bb6aa14f/172.17.0.2:0 2024-11-08T06:17:21,285 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-08T06:17:21,285 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-08T06:17:21,285 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-08T06:17:21,287 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3073): reportForDuty to master=d1f4bb6aa14f,46003,1731046638286 with isa=d1f4bb6aa14f/172.17.0.2:36803, startcode=1731046638998 2024-11-08T06:17:21,292 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-08T06:17:21,300 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-08T06:17:21,304 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T06:17:21,304 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T06:17:21,312 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d1f4bb6aa14f,46003,1731046638286 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T06:17:21,318 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d1f4bb6aa14f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T06:17:21,318 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d1f4bb6aa14f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T06:17:21,318 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d1f4bb6aa14f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T06:17:21,319 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d1f4bb6aa14f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T06:17:21,319 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d1f4bb6aa14f:0, corePoolSize=10, maxPoolSize=10 2024-11-08T06:17:21,320 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:21,320 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d1f4bb6aa14f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T06:17:21,320 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:21,339 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T06:17:21,340 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-08T06:17:21,344 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731046671344 2024-11-08T06:17:21,346 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:21,346 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T06:17:21,346 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T06:17:21,348 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T06:17:21,355 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T06:17:21,356 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T06:17:21,356 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T06:17:21,356 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T06:17:21,358 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:21,368 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T06:17:21,374 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T06:17:21,374 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42341, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T06:17:21,374 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T06:17:21,378 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T06:17:21,379 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T06:17:21,383 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46003 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3280) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:593) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T06:17:21,387 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d1f4bb6aa14f:0:becomeActiveMaster-HFileCleaner.large.0-1731046641381,5,FailOnTimeoutGroup] 2024-11-08T06:17:21,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741831_1007 (size=1039) 2024-11-08T06:17:21,395 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-08T06:17:21,395 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:17:21,404 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d1f4bb6aa14f:0:becomeActiveMaster-HFileCleaner.small.0-1731046641387,5,FailOnTimeoutGroup] 2024-11-08T06:17:21,404 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:21,404 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T06:17:21,406 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:21,406 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:21,417 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3097): Master is not running yet 2024-11-08T06:17:21,417 WARN [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1099): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-08T06:17:21,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741832_1008 (size=32) 2024-11-08T06:17:21,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:17:21,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T06:17:21,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T06:17:21,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:21,855 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T06:17:21,856 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T06:17:21,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T06:17:21,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:21,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T06:17:21,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T06:17:21,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T06:17:21,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:17:21,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T06:17:21,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740 2024-11-08T06:17:21,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740 2024-11-08T06:17:21,873 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T06:17:21,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T06:17:21,896 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:17:21,897 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60159273, jitterRate=-0.10355697572231293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T06:17:21,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T06:17:21,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T06:17:21,901 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T06:17:21,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T06:17:21,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T06:17:21,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T06:17:21,903 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T06:17:21,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T06:17:21,907 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T06:17:21,907 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-08T06:17:21,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T06:17:21,923 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T06:17:21,926 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T06:17:22,077 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:23,078 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:24,078 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:25,079 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:26,080 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:27,080 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:28,081 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:28,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T06:17:28,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T06:17:29,082 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:30,083 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:31,083 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:32,084 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:33,085 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:34,085 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:35,086 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:36,087 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:37,088 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:38,088 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:39,089 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:40,090 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:41,090 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:42,091 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:43,091 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:44,092 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:45,093 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:46,093 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:47,094 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:47,401 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:17:47,605 ERROR [LeaseRenewer:jenkins@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:45761,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:17:48,094 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:49,095 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:50,096 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:51,096 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:51,419 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3073): reportForDuty to master=d1f4bb6aa14f,46003,1731046638286 with isa=d1f4bb6aa14f/172.17.0.2:36803, startcode=1731046638998 2024-11-08T06:17:51,421 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46003 {}] master.ServerManager(332): Checking decommissioned status of RegionServer d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:17:51,424 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46003 {}] master.ServerManager(486): Registering regionserver=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:17:51,434 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:17:51,434 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:45761 2024-11-08T06:17:51,434 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-08T06:17:51,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T06:17:51,441 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] zookeeper.ZKUtil(111): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:17:51,441 WARN [RS:0;d1f4bb6aa14f:36803 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T06:17:51,441 INFO [RS:0;d1f4bb6aa14f:36803 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T06:17:51,442 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/WALs/d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:17:51,443 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d1f4bb6aa14f,36803,1731046638998] 2024-11-08T06:17:51,459 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-08T06:17:51,474 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T06:17:51,487 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T06:17:51,490 INFO [RS:0;d1f4bb6aa14f:36803 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T06:17:51,490 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:51,491 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-08T06:17:51,500 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:51,500 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,500 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,500 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,500 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,500 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,501 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d1f4bb6aa14f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T06:17:51,501 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,501 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,501 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,501 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,502 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d1f4bb6aa14f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T06:17:51,502 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d1f4bb6aa14f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T06:17:51,502 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d1f4bb6aa14f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T06:17:51,509 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:51,509 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:51,509 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:51,509 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:51,510 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,36803,1731046638998-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T06:17:51,540 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T06:17:51,543 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,36803,1731046638998-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T06:17:51,571 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.Replication(204): d1f4bb6aa14f,36803,1731046638998 started 2024-11-08T06:17:51,571 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1767): Serving as d1f4bb6aa14f,36803,1731046638998, RpcServer on d1f4bb6aa14f/172.17.0.2:36803, sessionid=0x1018686f4fc0001 2024-11-08T06:17:51,572 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T06:17:51,572 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:17:51,572 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1f4bb6aa14f,36803,1731046638998' 2024-11-08T06:17:51,572 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T06:17:51,573 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T06:17:51,574 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T06:17:51,574 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T06:17:51,574 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:17:51,574 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd1f4bb6aa14f,36803,1731046638998' 2024-11-08T06:17:51,574 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T06:17:51,575 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T06:17:51,575 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T06:17:51,576 INFO [RS:0;d1f4bb6aa14f:36803 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T06:17:51,576 INFO [RS:0;d1f4bb6aa14f:36803 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T06:17:52,097 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:53,098 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:54,099 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:55,099 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:55,282 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T06:17:55,286 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48064, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T06:17:56,101 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:57,102 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:58,102 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:17:59,103 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:00,104 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:01,105 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:02,105 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:03,106 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:04,107 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:05,107 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:06,108 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:07,109 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:08,109 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:09,110 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:10,111 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:11,111 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:12,112 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:13,113 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:14,113 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:15,114 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:16,115 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:17,115 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:17,402 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:18:18,116 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:19,117 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:20,117 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:21,118 WARN [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-08T06:18:21,581 INFO [RS:0;d1f4bb6aa14f:36803 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T06:18:21,585 INFO [RS:0;d1f4bb6aa14f:36803 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1f4bb6aa14f%2C36803%2C1731046638998, suffix=, logDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/WALs/d1f4bb6aa14f,36803,1731046638998, archiveDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/oldWALs, maxLogs=32 2024-11-08T06:18:21,604 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/WALs/d1f4bb6aa14f,36803,1731046638998/d1f4bb6aa14f%2C36803%2C1731046638998.1731046701589, exclude list is [], retry=0 2024-11-08T06:18:21,609 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35001,DS-502572dd-0ae8-4038-9112-bab8f9c64d4d,DISK] 2024-11-08T06:18:21,612 INFO [RS:0;d1f4bb6aa14f:36803 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/WALs/d1f4bb6aa14f,36803,1731046638998/d1f4bb6aa14f%2C36803%2C1731046638998.1731046701589 2024-11-08T06:18:21,612 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34773:34773)] 2024-11-08T06:18:21,870 DEBUG [d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T06:18:21,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:21,880 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1f4bb6aa14f,36803,1731046638998, state=OPENING 2024-11-08T06:18:21,885 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T06:18:21,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:18:21,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:18:21,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T06:18:21,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T06:18:21,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:22,063 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:22,065 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T06:18:22,067 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39460, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T06:18:22,078 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-08T06:18:22,078 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T06:18:22,078 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T06:18:22,082 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d1f4bb6aa14f%2C36803%2C1731046638998.meta, suffix=.meta, logDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/WALs/d1f4bb6aa14f,36803,1731046638998, archiveDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/oldWALs, maxLogs=32 2024-11-08T06:18:22,098 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/WALs/d1f4bb6aa14f,36803,1731046638998/d1f4bb6aa14f%2C36803%2C1731046638998.meta.1731046702084.meta, exclude list is [], retry=0 2024-11-08T06:18:22,102 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35001,DS-502572dd-0ae8-4038-9112-bab8f9c64d4d,DISK] 2024-11-08T06:18:22,105 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/WALs/d1f4bb6aa14f,36803,1731046638998/d1f4bb6aa14f%2C36803%2C1731046638998.meta.1731046702084.meta 2024-11-08T06:18:22,105 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34773:34773)] 2024-11-08T06:18:22,105 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:22,107 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T06:18:22,166 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T06:18:22,170 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T06:18:22,174 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T06:18:22,174 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:22,174 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-08T06:18:22,175 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-08T06:18:22,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T06:18:22,180 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T06:18:22,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:22,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T06:18:22,182 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T06:18:22,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T06:18:22,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:22,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T06:18:22,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T06:18:22,187 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T06:18:22,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:22,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T06:18:22,190 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740 2024-11-08T06:18:22,193 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740 2024-11-08T06:18:22,196 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T06:18:22,199 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T06:18:22,200 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61999484, jitterRate=-0.07613569498062134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T06:18:22,202 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T06:18:22,209 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731046702058 2024-11-08T06:18:22,222 DEBUG [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T06:18:22,223 INFO [RS_OPEN_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-08T06:18:22,224 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:22,226 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d1f4bb6aa14f,36803,1731046638998, state=OPEN 2024-11-08T06:18:22,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T06:18:22,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T06:18:22,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T06:18:22,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T06:18:22,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T06:18:22,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=d1f4bb6aa14f,36803,1731046638998 in 341 msec 2024-11-08T06:18:22,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T06:18:22,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.322 sec 2024-11-08T06:18:22,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1 mins, 1.017 sec 2024-11-08T06:18:22,244 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731046702244, completionTime=-1 2024-11-08T06:18:22,244 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T06:18:22,245 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-08T06:18:22,281 DEBUG [hconnection-0x60cdbea-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:22,283 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39464, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:22,294 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-08T06:18:22,294 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731046762294 2024-11-08T06:18:22,294 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731046822294 2024-11-08T06:18:22,294 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 49 msec 2024-11-08T06:18:22,313 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,46003,1731046638286-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T06:18:22,313 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,46003,1731046638286-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T06:18:22,313 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,46003,1731046638286-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T06:18:22,315 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d1f4bb6aa14f:46003, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T06:18:22,315 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T06:18:22,320 DEBUG [master/d1f4bb6aa14f:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-08T06:18:22,323 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-08T06:18:22,324 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T06:18:22,330 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-08T06:18:22,333 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:22,334 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:22,335 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:22,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741835_1011 (size=358) 2024-11-08T06:18:22,751 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 640f50ffb597620ddc4800f241b7ca8f, NAME => 'hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:22,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741836_1012 (size=42) 2024-11-08T06:18:23,161 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:23,162 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 640f50ffb597620ddc4800f241b7ca8f, disabling compactions & flushes 2024-11-08T06:18:23,162 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:18:23,162 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:18:23,162 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. after waiting 0 ms 2024-11-08T06:18:23,162 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:18:23,162 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:18:23,162 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 640f50ffb597620ddc4800f241b7ca8f: 2024-11-08T06:18:23,164 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:23,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731046703166"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046703166"}]},"ts":"1731046703166"} 2024-11-08T06:18:23,195 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:23,196 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:23,199 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046703197"}]},"ts":"1731046703197"} 2024-11-08T06:18:23,203 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-08T06:18:23,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=640f50ffb597620ddc4800f241b7ca8f, ASSIGN}] 2024-11-08T06:18:23,211 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=640f50ffb597620ddc4800f241b7ca8f, ASSIGN 2024-11-08T06:18:23,213 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=640f50ffb597620ddc4800f241b7ca8f, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:23,363 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=640f50ffb597620ddc4800f241b7ca8f, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:23,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 640f50ffb597620ddc4800f241b7ca8f, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:23,521 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:23,527 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:18:23,527 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 640f50ffb597620ddc4800f241b7ca8f, NAME => 'hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:23,528 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:18:23,528 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:23,528 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:18:23,528 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:18:23,530 INFO [StoreOpener-640f50ffb597620ddc4800f241b7ca8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:18:23,533 INFO [StoreOpener-640f50ffb597620ddc4800f241b7ca8f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 640f50ffb597620ddc4800f241b7ca8f columnFamilyName info 2024-11-08T06:18:23,533 DEBUG [StoreOpener-640f50ffb597620ddc4800f241b7ca8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:23,534 INFO [StoreOpener-640f50ffb597620ddc4800f241b7ca8f-1 {}] regionserver.HStore(327): Store=640f50ffb597620ddc4800f241b7ca8f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:23,535 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:18:23,536 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:18:23,541 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:18:23,546 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:23,547 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 640f50ffb597620ddc4800f241b7ca8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60019815, jitterRate=-0.10563506186008453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:23,548 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 640f50ffb597620ddc4800f241b7ca8f: 2024-11-08T06:18:23,550 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f., pid=6, masterSystemTime=1731046703521 2024-11-08T06:18:23,554 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:18:23,554 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:18:23,555 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=640f50ffb597620ddc4800f241b7ca8f, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:23,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T06:18:23,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 640f50ffb597620ddc4800f241b7ca8f, server=d1f4bb6aa14f,36803,1731046638998 in 191 msec 2024-11-08T06:18:23,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T06:18:23,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=640f50ffb597620ddc4800f241b7ca8f, ASSIGN in 354 msec 2024-11-08T06:18:23,568 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:23,568 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046703568"}]},"ts":"1731046703568"} 2024-11-08T06:18:23,573 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-08T06:18:23,577 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:23,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2520 sec 2024-11-08T06:18:23,634 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-08T06:18:23,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-08T06:18:23,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:18:23,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:18:23,689 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-08T06:18:23,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T06:18:23,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 50 msec 2024-11-08T06:18:23,726 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-08T06:18:23,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T06:18:23,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 17 msec 2024-11-08T06:18:23,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-08T06:18:23,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-08T06:18:23,755 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 64.683sec 2024-11-08T06:18:23,757 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T06:18:23,758 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T06:18:23,759 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T06:18:23,760 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T06:18:23,760 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T06:18:23,761 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,46003,1731046638286-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T06:18:23,762 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,46003,1731046638286-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T06:18:23,769 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-08T06:18:23,770 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T06:18:23,770 INFO [master/d1f4bb6aa14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d1f4bb6aa14f,46003,1731046638286-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T06:18:23,798 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x574a453f to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61e4a9fc 2024-11-08T06:18:23,798 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-08T06:18:23,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d933e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:23,809 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T06:18:23,809 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T06:18:23,818 DEBUG [hconnection-0x35085285-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:23,826 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39474, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:23,834 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:18:23,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=214, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=3236 2024-11-08T06:18:23,855 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x00d5915a to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a8ef6a9 2024-11-08T06:18:23,860 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x00d5915a {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc3f18d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:23,864 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c2cf168 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@312a75fa 2024-11-08T06:18:23,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42bb6992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:23,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T06:18:23,877 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49014, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T06:18:23,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:18:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T06:18:23,889 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:23,890 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:23,891 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:23,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-08T06:18:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T06:18:23,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741837_1013 (size=437) 2024-11-08T06:18:24,304 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4452a1cba67ce3322ca17c876abcd582, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:24,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741838_1014 (size=88) 2024-11-08T06:18:24,316 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:24,316 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 4452a1cba67ce3322ca17c876abcd582, disabling compactions & flushes 2024-11-08T06:18:24,316 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:18:24,317 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:18:24,317 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. after waiting 0 ms 2024-11-08T06:18:24,317 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:18:24,317 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:18:24,317 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4452a1cba67ce3322ca17c876abcd582: 2024-11-08T06:18:24,319 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:24,319 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731046704319"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046704319"}]},"ts":"1731046704319"} 2024-11-08T06:18:24,323 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:24,324 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:24,325 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046704324"}]},"ts":"1731046704324"} 2024-11-08T06:18:24,328 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:18:24,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=4452a1cba67ce3322ca17c876abcd582, ASSIGN}] 2024-11-08T06:18:24,335 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=4452a1cba67ce3322ca17c876abcd582, ASSIGN 2024-11-08T06:18:24,336 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=4452a1cba67ce3322ca17c876abcd582, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:24,487 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=4452a1cba67ce3322ca17c876abcd582, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:24,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 4452a1cba67ce3322ca17c876abcd582, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:24,645 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:24,652 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:18:24,652 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 4452a1cba67ce3322ca17c876abcd582, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:24,653 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false 4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:18:24,653 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:24,653 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:18:24,653 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:18:24,655 INFO [StoreOpener-4452a1cba67ce3322ca17c876abcd582-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:18:24,657 INFO [StoreOpener-4452a1cba67ce3322ca17c876abcd582-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4452a1cba67ce3322ca17c876abcd582 columnFamilyName testFamily 2024-11-08T06:18:24,657 DEBUG [StoreOpener-4452a1cba67ce3322ca17c876abcd582-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:24,658 INFO [StoreOpener-4452a1cba67ce3322ca17c876abcd582-1 {}] regionserver.HStore(327): Store=4452a1cba67ce3322ca17c876abcd582/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:24,659 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:18:24,660 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:18:24,664 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:18:24,668 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:24,669 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 4452a1cba67ce3322ca17c876abcd582; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59471510, jitterRate=-0.11380544304847717}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:24,670 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 4452a1cba67ce3322ca17c876abcd582: 2024-11-08T06:18:24,671 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582., pid=11, masterSystemTime=1731046704644 2024-11-08T06:18:24,674 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:18:24,674 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:18:24,676 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=4452a1cba67ce3322ca17c876abcd582, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:24,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T06:18:24,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 4452a1cba67ce3322ca17c876abcd582, server=d1f4bb6aa14f,36803,1731046638998 in 188 msec 2024-11-08T06:18:24,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T06:18:24,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=4452a1cba67ce3322ca17c876abcd582, ASSIGN in 349 msec 2024-11-08T06:18:24,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:24,687 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046704687"}]},"ts":"1731046704687"} 2024-11-08T06:18:24,689 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:18:24,693 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:24,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 808 msec 2024-11-08T06:18:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T06:18:24,902 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-08T06:18:24,902 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:18:24,903 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:24,908 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T06:18:24,908 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:24,908 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-08T06:18:24,911 DEBUG [hconnection-0x1224ccda-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:24,913 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39486, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:24,925 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:18:24,930 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T06:18:24,942 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:18:24,944 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x00d5915a {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:18:24,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:24,950 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39498, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:18:24,959 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T06:18:24,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567307 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:24,961 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T06:18:24,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567307 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:24,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567307 2024-11-08T06:18:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:39498 deadline: 1731046707961, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:24,964 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567307 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:18:24,965 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T06:18:24,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567308 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:24,967 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T06:18:24,967 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:24,968 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:24,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567309 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:24,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567309 2024-11-08T06:18:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:39498 deadline: 1731046707967, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:24,970 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567309 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:18:24,971 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:24,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567310 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:24,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=8767934147168567310 2024-11-08T06:18:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:39498 deadline: 1731046707970, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:24,974 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:24,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567311 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:24,973 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567310 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:18:24,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=8767934147168567311 2024-11-08T06:18:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:39498 deadline: 1731046707973, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:24,977 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567311 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:18:24,978 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:24,978 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c2cf168 to 127.0.0.1:49997 2024-11-08T06:18:24,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:24,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:18:24,979 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:18:24,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:24,984 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x00d5915a to 127.0.0.1:49997 2024-11-08T06:18:24,994 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=227 (was 214) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49997@0x0c2cf168-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49997@0x00d5915a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x35085285-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49997@0x00d5915a java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$739/0x00007f8c90b478c8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49997@0x0c2cf168 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$739/0x00007f8c90b478c8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2083072951_22 at /127.0.0.1:43754 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49997@0x00d5915a-SendThread(127.0.0.1:49997) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:49997@0x0c2cf168-SendThread(127.0.0.1:49997) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) - Thread LEAK? -, OpenFileDescriptor=456 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=439 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=3217 (was 3236) 2024-11-08T06:18:25,002 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=227, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=439, ProcessCount=11, AvailableMemoryMB=3217 2024-11-08T06:18:25,004 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x790bfaf9 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6222913c 2024-11-08T06:18:25,007 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x790bfaf9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c97564e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:25,009 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2470e2cd to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18ac8fdd 2024-11-08T06:18:25,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4e4ad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:25,015 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:18:25,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T06:18:25,018 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:25,018 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:25,018 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-08T06:18:25,020 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T06:18:25,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741839_1015 (size=423) 2024-11-08T06:18:25,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bef1d5c4b90cf2bf782264d6131b3426, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:25,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741840_1016 (size=74) 2024-11-08T06:18:25,842 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:25,842 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing bef1d5c4b90cf2bf782264d6131b3426, disabling compactions & flushes 2024-11-08T06:18:25,842 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:18:25,842 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:18:25,842 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. after waiting 0 ms 2024-11-08T06:18:25,842 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:18:25,842 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:18:25,842 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for bef1d5c4b90cf2bf782264d6131b3426: 2024-11-08T06:18:25,845 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:25,845 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731046705845"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046705845"}]},"ts":"1731046705845"} 2024-11-08T06:18:25,848 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:25,850 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:25,850 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046705850"}]},"ts":"1731046705850"} 2024-11-08T06:18:25,853 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:18:25,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=bef1d5c4b90cf2bf782264d6131b3426, ASSIGN}] 2024-11-08T06:18:25,859 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=bef1d5c4b90cf2bf782264d6131b3426, ASSIGN 2024-11-08T06:18:25,861 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=bef1d5c4b90cf2bf782264d6131b3426, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:26,011 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=bef1d5c4b90cf2bf782264d6131b3426, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:26,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure bef1d5c4b90cf2bf782264d6131b3426, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T06:18:26,168 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:26,174 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:18:26,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => bef1d5c4b90cf2bf782264d6131b3426, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:26,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:18:26,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:26,175 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:18:26,175 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:18:26,177 INFO [StoreOpener-bef1d5c4b90cf2bf782264d6131b3426-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:18:26,179 INFO [StoreOpener-bef1d5c4b90cf2bf782264d6131b3426-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bef1d5c4b90cf2bf782264d6131b3426 columnFamilyName testFamily 2024-11-08T06:18:26,179 DEBUG [StoreOpener-bef1d5c4b90cf2bf782264d6131b3426-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:26,180 INFO [StoreOpener-bef1d5c4b90cf2bf782264d6131b3426-1 {}] regionserver.HStore(327): Store=bef1d5c4b90cf2bf782264d6131b3426/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:26,181 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:18:26,182 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:18:26,185 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:18:26,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:26,189 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened bef1d5c4b90cf2bf782264d6131b3426; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66686601, jitterRate=-0.006292209029197693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:26,190 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for bef1d5c4b90cf2bf782264d6131b3426: 2024-11-08T06:18:26,191 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426., pid=14, masterSystemTime=1731046706168 2024-11-08T06:18:26,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:18:26,195 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:18:26,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=bef1d5c4b90cf2bf782264d6131b3426, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:26,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T06:18:26,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure bef1d5c4b90cf2bf782264d6131b3426, server=d1f4bb6aa14f,36803,1731046638998 in 183 msec 2024-11-08T06:18:26,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-08T06:18:26,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=bef1d5c4b90cf2bf782264d6131b3426, ASSIGN in 344 msec 2024-11-08T06:18:26,205 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:26,206 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046706205"}]},"ts":"1731046706205"} 2024-11-08T06:18:26,208 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:18:26,211 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:26,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 1.1960 sec 2024-11-08T06:18:27,722 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:18:27,759 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-08T06:18:27,761 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T06:18:27,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-08T06:18:27,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-08T06:18:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T06:18:28,026 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-08T06:18:28,026 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:18:28,026 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:28,033 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T06:18:28,033 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:28,033 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T06:18:28,036 DEBUG [hconnection-0x1648742b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:28,039 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39508, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:28,061 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:18:28,065 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:18:28,067 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:18:28,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T06:18:28,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:28,773 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T06:18:28,773 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:28,775 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T06:18:28,775 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:28,776 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T06:18:28,776 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T06:18:28,776 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T06:18:28,776 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T06:18:35,090 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:18:37,069 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11005: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11005: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:18:37,070 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T06:18:37.070Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11005: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11005: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:18:37,071 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-08T06:18:37,072 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2470e2cd to 127.0.0.1:49997 2024-11-08T06:18:37,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:37,072 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:18:37,072 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:18:37,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:37,073 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x790bfaf9 to 127.0.0.1:49997 2024-11-08T06:18:37,086 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=216 (was 227), OpenFileDescriptor=450 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 439), ProcessCount=11 (was 11), AvailableMemoryMB=3184 (was 3217) 2024-11-08T06:18:37,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=216, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=3184 2024-11-08T06:18:37,108 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27ceb32f to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15edec57 2024-11-08T06:18:37,113 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x27ceb32f {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dd0051b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:37,115 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x509ba719 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@798b6001 2024-11-08T06:18:37,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8ee3d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:37,124 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:18:37,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T06:18:37,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:37,128 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:37,128 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-08T06:18:37,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T06:18:37,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741841_1017 (size=421) 2024-11-08T06:18:37,161 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4f990b745771e77a5b0a8093dac6553c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:37,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741842_1018 (size=72) 2024-11-08T06:18:37,178 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:37,178 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 4f990b745771e77a5b0a8093dac6553c, disabling compactions & flushes 2024-11-08T06:18:37,178 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:18:37,178 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:18:37,178 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. after waiting 0 ms 2024-11-08T06:18:37,178 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:18:37,178 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:18:37,178 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4f990b745771e77a5b0a8093dac6553c: 2024-11-08T06:18:37,180 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:37,180 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731046717180"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046717180"}]},"ts":"1731046717180"} 2024-11-08T06:18:37,184 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:37,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:37,186 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046717186"}]},"ts":"1731046717186"} 2024-11-08T06:18:37,188 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:18:37,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4f990b745771e77a5b0a8093dac6553c, ASSIGN}] 2024-11-08T06:18:37,199 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4f990b745771e77a5b0a8093dac6553c, ASSIGN 2024-11-08T06:18:37,200 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4f990b745771e77a5b0a8093dac6553c, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:37,351 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=4f990b745771e77a5b0a8093dac6553c, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:37,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 4f990b745771e77a5b0a8093dac6553c, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:37,509 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:37,524 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:18:37,525 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 4f990b745771e77a5b0a8093dac6553c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:37,525 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:18:37,525 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:37,525 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:18:37,525 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:18:37,527 INFO [StoreOpener-4f990b745771e77a5b0a8093dac6553c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:18:37,530 INFO [StoreOpener-4f990b745771e77a5b0a8093dac6553c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f990b745771e77a5b0a8093dac6553c columnFamilyName testFamily 2024-11-08T06:18:37,530 DEBUG [StoreOpener-4f990b745771e77a5b0a8093dac6553c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:37,531 INFO [StoreOpener-4f990b745771e77a5b0a8093dac6553c-1 {}] regionserver.HStore(327): Store=4f990b745771e77a5b0a8093dac6553c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:37,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:18:37,533 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:18:37,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:18:37,548 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:37,549 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened 4f990b745771e77a5b0a8093dac6553c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74975636, jitterRate=0.11722403764724731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:37,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 4f990b745771e77a5b0a8093dac6553c: 2024-11-08T06:18:37,551 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c., pid=17, masterSystemTime=1731046717509 2024-11-08T06:18:37,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:18:37,555 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:18:37,556 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=4f990b745771e77a5b0a8093dac6553c, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:37,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-08T06:18:37,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 4f990b745771e77a5b0a8093dac6553c, server=d1f4bb6aa14f,36803,1731046638998 in 204 msec 2024-11-08T06:18:37,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T06:18:37,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4f990b745771e77a5b0a8093dac6553c, ASSIGN in 367 msec 2024-11-08T06:18:37,568 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:37,568 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046717568"}]},"ts":"1731046717568"} 2024-11-08T06:18:37,570 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:18:37,574 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:37,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 450 msec 2024-11-08T06:18:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-08T06:18:38,135 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-08T06:18:38,135 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:18:38,135 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:38,140 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T06:18:38,141 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:38,141 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T06:18:38,143 DEBUG [hconnection-0x2230cf54-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:38,146 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58648, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:38,161 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:18:38,165 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:18:38,165 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:18:38,167 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x27ceb32f {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:18:38,167 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x27ceb32f {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:38,170 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58654, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:18:38,175 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567323 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:38,176 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:18:38,177 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:18:38,176 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567323 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:38,177 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T06:18:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T06:18:38,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:39,764 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-08T06:18:41,591 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:18:43,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:18:47,179 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9002 2024-11-08T06:18:47,180 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x509ba719 to 127.0.0.1:49997 2024-11-08T06:18:47,179 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567323 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:18:47,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:47,180 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:18:47,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567323' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T06:18:47,180 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:18:47,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:58654 deadline: 1731046733179, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567323' does not exist 2024-11-08T06:18:47,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:47,180 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x27ceb32f to 127.0.0.1:49997 2024-11-08T06:18:47,181 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d1f4bb6aa14f,36803,1731046638998 for closing scanner id = 8767934147168567323 for 4f990b745771e77a5b0a8093dac6553c of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T06:18:47,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=220 (was 216) - Thread LEAK? -, OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 402) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3118 (was 3184) 2024-11-08T06:18:47,199 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=220, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=3118 2024-11-08T06:18:47,200 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69f0ffa1 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a952052 2024-11-08T06:18:47,204 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x69f0ffa1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fc445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:47,205 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5228f3c8 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@75859c3c 2024-11-08T06:18:47,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6678da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:47,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:18:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T06:18:47,215 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:47,215 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:47,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-08T06:18:47,216 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T06:18:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741843_1019 (size=432) 2024-11-08T06:18:47,402 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:18:47,629 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2a97b1538ceac9f3bb41ab032e3eaa06, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741844_1020 (size=83) 2024-11-08T06:18:47,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:47,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2a97b1538ceac9f3bb41ab032e3eaa06, disabling compactions & flushes 2024-11-08T06:18:47,639 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:18:47,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:18:47,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. after waiting 0 ms 2024-11-08T06:18:47,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:18:47,639 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:18:47,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2a97b1538ceac9f3bb41ab032e3eaa06: 2024-11-08T06:18:47,640 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:47,641 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731046727641"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046727641"}]},"ts":"1731046727641"} 2024-11-08T06:18:47,644 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:47,645 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:47,646 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046727645"}]},"ts":"1731046727645"} 2024-11-08T06:18:47,648 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:18:47,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=2a97b1538ceac9f3bb41ab032e3eaa06, ASSIGN}] 2024-11-08T06:18:47,659 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=2a97b1538ceac9f3bb41ab032e3eaa06, ASSIGN 2024-11-08T06:18:47,660 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=2a97b1538ceac9f3bb41ab032e3eaa06, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:47,678 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:18:47,811 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=2a97b1538ceac9f3bb41ab032e3eaa06, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:47,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure 2a97b1538ceac9f3bb41ab032e3eaa06, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:47,967 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:47,974 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:18:47,974 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => 2a97b1538ceac9f3bb41ab032e3eaa06, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:47,975 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:18:47,975 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:47,975 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:18:47,975 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:18:47,977 INFO [StoreOpener-2a97b1538ceac9f3bb41ab032e3eaa06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:18:47,979 INFO [StoreOpener-2a97b1538ceac9f3bb41ab032e3eaa06-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a97b1538ceac9f3bb41ab032e3eaa06 columnFamilyName testFamily 2024-11-08T06:18:47,979 DEBUG [StoreOpener-2a97b1538ceac9f3bb41ab032e3eaa06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:47,980 INFO [StoreOpener-2a97b1538ceac9f3bb41ab032e3eaa06-1 {}] regionserver.HStore(327): Store=2a97b1538ceac9f3bb41ab032e3eaa06/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:47,981 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:18:47,982 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:18:47,985 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:18:47,989 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:47,990 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened 2a97b1538ceac9f3bb41ab032e3eaa06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63719854, jitterRate=-0.050500184297561646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:47,991 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for 2a97b1538ceac9f3bb41ab032e3eaa06: 2024-11-08T06:18:47,992 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06., pid=20, masterSystemTime=1731046727967 2024-11-08T06:18:47,994 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:18:47,995 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:18:47,995 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=2a97b1538ceac9f3bb41ab032e3eaa06, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:48,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T06:18:48,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure 2a97b1538ceac9f3bb41ab032e3eaa06, server=d1f4bb6aa14f,36803,1731046638998 in 183 msec 2024-11-08T06:18:48,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-08T06:18:48,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=2a97b1538ceac9f3bb41ab032e3eaa06, ASSIGN in 343 msec 2024-11-08T06:18:48,004 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:48,004 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046728004"}]},"ts":"1731046728004"} 2024-11-08T06:18:48,006 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:18:48,010 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:48,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 798 msec 2024-11-08T06:18:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T06:18:48,219 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-08T06:18:48,219 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:18:48,219 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:48,223 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-08T06:18:48,224 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:48,224 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-08T06:18:48,226 DEBUG [hconnection-0x763e6210-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:48,228 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40310, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:48,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:18:48,240 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T06:18:48,242 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T06:18:48,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567327 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:48,243 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T06:18:48,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567327 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:48,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567327 2024-11-08T06:18:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:40310 deadline: 1731046729244, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:48,245 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:18:48,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T06:18:48,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567328 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:48,248 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T06:18:48,248 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:48,249 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:48,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567329 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:48,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567329 2024-11-08T06:18:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:40310 deadline: 1731046729250, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:48,251 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:18:48,252 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:48,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567330 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:48,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=8767934147168567330 2024-11-08T06:18:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:40310 deadline: 1731046729252, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:48,253 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:18:48,255 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:48,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567331 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:48,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=8767934147168567331 2024-11-08T06:18:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:40310 deadline: 1731046729255, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:18:48,256 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:18:48,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:18:48,258 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5228f3c8 to 127.0.0.1:49997 2024-11-08T06:18:48,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:48,258 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:18:48,258 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:18:48,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:48,259 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69f0ffa1 to 127.0.0.1:49997 2024-11-08T06:18:48,268 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=226 (was 220) - Thread LEAK? -, OpenFileDescriptor=454 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=3116 (was 3118) 2024-11-08T06:18:48,277 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=3116 2024-11-08T06:18:48,278 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1824014e to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fdc096e 2024-11-08T06:18:48,281 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1824014e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f3296b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:48,283 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66b76e83 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a9c4477 2024-11-08T06:18:48,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428a041b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:48,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:18:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T06:18:48,291 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:48,291 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:48,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-08T06:18:48,292 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T06:18:48,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741845_1021 (size=426) 2024-11-08T06:18:48,303 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e2211a6e8e961230cc6dbdd2a2186983, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:48,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741846_1022 (size=77) 2024-11-08T06:18:48,713 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:48,713 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing e2211a6e8e961230cc6dbdd2a2186983, disabling compactions & flushes 2024-11-08T06:18:48,713 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:18:48,713 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:18:48,713 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. after waiting 0 ms 2024-11-08T06:18:48,713 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:18:48,713 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:18:48,713 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for e2211a6e8e961230cc6dbdd2a2186983: 2024-11-08T06:18:48,715 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:48,716 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731046728715"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046728715"}]},"ts":"1731046728715"} 2024-11-08T06:18:48,718 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:48,719 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:48,720 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046728719"}]},"ts":"1731046728719"} 2024-11-08T06:18:48,722 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:18:48,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=e2211a6e8e961230cc6dbdd2a2186983, ASSIGN}] 2024-11-08T06:18:48,729 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=e2211a6e8e961230cc6dbdd2a2186983, ASSIGN 2024-11-08T06:18:48,730 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=e2211a6e8e961230cc6dbdd2a2186983, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:48,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-08T06:18:48,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:48,881 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=e2211a6e8e961230cc6dbdd2a2186983, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:48,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure e2211a6e8e961230cc6dbdd2a2186983, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:49,037 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:49,042 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:18:49,043 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => e2211a6e8e961230cc6dbdd2a2186983, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:49,043 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:18:49,043 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:49,043 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:18:49,044 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:18:49,045 INFO [StoreOpener-e2211a6e8e961230cc6dbdd2a2186983-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:18:49,048 INFO [StoreOpener-e2211a6e8e961230cc6dbdd2a2186983-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2211a6e8e961230cc6dbdd2a2186983 columnFamilyName testFamily 2024-11-08T06:18:49,048 DEBUG [StoreOpener-e2211a6e8e961230cc6dbdd2a2186983-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:49,049 INFO [StoreOpener-e2211a6e8e961230cc6dbdd2a2186983-1 {}] regionserver.HStore(327): Store=e2211a6e8e961230cc6dbdd2a2186983/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:49,050 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:18:49,050 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:18:49,053 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:18:49,055 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:49,056 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened e2211a6e8e961230cc6dbdd2a2186983; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65245678, jitterRate=-0.02776363492012024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:49,057 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for e2211a6e8e961230cc6dbdd2a2186983: 2024-11-08T06:18:49,058 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983., pid=23, masterSystemTime=1731046729037 2024-11-08T06:18:49,060 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:18:49,060 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:18:49,061 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=e2211a6e8e961230cc6dbdd2a2186983, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:49,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-08T06:18:49,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure e2211a6e8e961230cc6dbdd2a2186983, server=d1f4bb6aa14f,36803,1731046638998 in 179 msec 2024-11-08T06:18:49,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T06:18:49,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=e2211a6e8e961230cc6dbdd2a2186983, ASSIGN in 337 msec 2024-11-08T06:18:49,068 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:49,069 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046729068"}]},"ts":"1731046729068"} 2024-11-08T06:18:49,070 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:18:49,074 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:49,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 786 msec 2024-11-08T06:18:49,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-08T06:18:49,295 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-08T06:18:49,295 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:18:49,295 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:49,299 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T06:18:49,299 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:49,299 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T06:18:49,301 DEBUG [hconnection-0x5a53605a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:49,303 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40320, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:49,313 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:18:49,316 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:18:49,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:18:49,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T06:18:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T06:18:51,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:18:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:40320 deadline: 1731046732318, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:18:51,344 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:18:51,345 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567336 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:18:51,347 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2030 2024-11-08T06:18:51,347 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66b76e83 to 127.0.0.1:49997 2024-11-08T06:18:51,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:51,348 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:18:51,348 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:18:51,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:51,348 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1824014e to 127.0.0.1:49997 2024-11-08T06:18:51,359 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=227 (was 226) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=3118 (was 3116) - AvailableMemoryMB LEAK? - 2024-11-08T06:18:51,369 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=227, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=3117 2024-11-08T06:18:51,371 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58ce1d85 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45a8adb1 2024-11-08T06:18:51,375 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x58ce1d85 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c592180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:51,376 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11a8dd84 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41a77621 2024-11-08T06:18:51,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efb3040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:51,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:18:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T06:18:51,385 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:51,385 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:51,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-08T06:18:51,386 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T06:18:51,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741847_1023 (size=427) 2024-11-08T06:18:51,396 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2cb28dcad64759a3f944589ee2bafd95, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:51,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741848_1024 (size=78) 2024-11-08T06:18:51,609 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:45761,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:18:51,767 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-08T06:18:51,767 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-08T06:18:51,807 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:51,807 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2cb28dcad64759a3f944589ee2bafd95, disabling compactions & flushes 2024-11-08T06:18:51,807 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:18:51,807 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:18:51,807 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. after waiting 0 ms 2024-11-08T06:18:51,807 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:18:51,807 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:18:51,807 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2cb28dcad64759a3f944589ee2bafd95: 2024-11-08T06:18:51,809 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:51,809 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731046731809"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046731809"}]},"ts":"1731046731809"} 2024-11-08T06:18:51,812 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:51,813 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:51,813 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046731813"}]},"ts":"1731046731813"} 2024-11-08T06:18:51,816 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:18:51,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=2cb28dcad64759a3f944589ee2bafd95, ASSIGN}] 2024-11-08T06:18:51,822 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=2cb28dcad64759a3f944589ee2bafd95, ASSIGN 2024-11-08T06:18:51,823 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=2cb28dcad64759a3f944589ee2bafd95, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:51,973 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=2cb28dcad64759a3f944589ee2bafd95, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:51,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 2cb28dcad64759a3f944589ee2bafd95, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:52,129 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:52,134 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:18:52,134 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => 2cb28dcad64759a3f944589ee2bafd95, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:52,135 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:18:52,135 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:52,135 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for 2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:18:52,135 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for 2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:18:52,137 INFO [StoreOpener-2cb28dcad64759a3f944589ee2bafd95-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:18:52,139 INFO [StoreOpener-2cb28dcad64759a3f944589ee2bafd95-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cb28dcad64759a3f944589ee2bafd95 columnFamilyName testFamily 2024-11-08T06:18:52,139 DEBUG [StoreOpener-2cb28dcad64759a3f944589ee2bafd95-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:52,139 INFO [StoreOpener-2cb28dcad64759a3f944589ee2bafd95-1 {}] regionserver.HStore(327): Store=2cb28dcad64759a3f944589ee2bafd95/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:52,140 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:18:52,141 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:18:52,144 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for 2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:18:52,147 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:52,148 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened 2cb28dcad64759a3f944589ee2bafd95; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62839055, jitterRate=-0.06362511217594147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:52,148 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for 2cb28dcad64759a3f944589ee2bafd95: 2024-11-08T06:18:52,149 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95., pid=26, masterSystemTime=1731046732129 2024-11-08T06:18:52,152 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:18:52,152 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:18:52,152 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=2cb28dcad64759a3f944589ee2bafd95, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:52,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T06:18:52,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 2cb28dcad64759a3f944589ee2bafd95, server=d1f4bb6aa14f,36803,1731046638998 in 179 msec 2024-11-08T06:18:52,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-08T06:18:52,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=2cb28dcad64759a3f944589ee2bafd95, ASSIGN in 337 msec 2024-11-08T06:18:52,160 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:52,161 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046732161"}]},"ts":"1731046732161"} 2024-11-08T06:18:52,162 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:18:52,165 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:52,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 783 msec 2024-11-08T06:18:52,344 ERROR [LeaseRenewer:jenkins@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:45761,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:18:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T06:18:52,389 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-08T06:18:52,389 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:18:52,390 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:52,394 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-08T06:18:52,395 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:52,395 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-08T06:18:52,397 DEBUG [hconnection-0xc8e56da-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:52,399 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39168, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:52,409 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:18:52,413 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:18:52,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:18:53,477 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:18:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T06:18:54,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:18:55,419 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:18:55,419 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T06:18:55.419Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5010: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:18:55,420 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3007 2024-11-08T06:18:55,421 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x11a8dd84 to 127.0.0.1:49997 2024-11-08T06:18:55,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:55,421 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:18:55,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:18:55,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:18:55,421 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58ce1d85 to 127.0.0.1:49997 2024-11-08T06:18:55,432 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225 (was 227), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 426) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3115 (was 3117) 2024-11-08T06:18:55,440 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=3115 2024-11-08T06:18:55,442 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5176280c to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4361dc64 2024-11-08T06:18:55,446 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x5176280c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ecb2e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:55,447 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58a43e93 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d532004 2024-11-08T06:18:55,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0541ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:18:55,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:18:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T06:18:55,456 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:18:55,456 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:55,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-08T06:18:55,458 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:18:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T06:18:55,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741849_1025 (size=425) 2024-11-08T06:18:55,868 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 169fe32fc4194690caf6575dbee1d411, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:18:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741850_1026 (size=76) 2024-11-08T06:18:55,929 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:18:56,277 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:56,277 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 169fe32fc4194690caf6575dbee1d411, disabling compactions & flushes 2024-11-08T06:18:56,277 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:18:56,277 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:18:56,277 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. after waiting 0 ms 2024-11-08T06:18:56,277 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:18:56,277 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:18:56,277 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 169fe32fc4194690caf6575dbee1d411: 2024-11-08T06:18:56,280 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:18:56,280 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731046736280"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046736280"}]},"ts":"1731046736280"} 2024-11-08T06:18:56,283 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:18:56,284 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:18:56,285 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046736284"}]},"ts":"1731046736284"} 2024-11-08T06:18:56,286 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:18:56,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=169fe32fc4194690caf6575dbee1d411, ASSIGN}] 2024-11-08T06:18:56,294 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=169fe32fc4194690caf6575dbee1d411, ASSIGN 2024-11-08T06:18:56,296 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=169fe32fc4194690caf6575dbee1d411, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:18:56,447 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=169fe32fc4194690caf6575dbee1d411, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:56,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure 169fe32fc4194690caf6575dbee1d411, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:18:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T06:18:56,607 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:56,613 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:18:56,614 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => 169fe32fc4194690caf6575dbee1d411, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:18:56,614 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false 169fe32fc4194690caf6575dbee1d411 2024-11-08T06:18:56,614 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:18:56,614 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for 169fe32fc4194690caf6575dbee1d411 2024-11-08T06:18:56,615 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for 169fe32fc4194690caf6575dbee1d411 2024-11-08T06:18:56,617 INFO [StoreOpener-169fe32fc4194690caf6575dbee1d411-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 169fe32fc4194690caf6575dbee1d411 2024-11-08T06:18:56,620 INFO [StoreOpener-169fe32fc4194690caf6575dbee1d411-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 169fe32fc4194690caf6575dbee1d411 columnFamilyName testFamily 2024-11-08T06:18:56,620 DEBUG [StoreOpener-169fe32fc4194690caf6575dbee1d411-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:18:56,621 INFO [StoreOpener-169fe32fc4194690caf6575dbee1d411-1 {}] regionserver.HStore(327): Store=169fe32fc4194690caf6575dbee1d411/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:18:56,622 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411 2024-11-08T06:18:56,623 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411 2024-11-08T06:18:56,626 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for 169fe32fc4194690caf6575dbee1d411 2024-11-08T06:18:56,631 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:18:56,633 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened 169fe32fc4194690caf6575dbee1d411; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66848971, jitterRate=-0.003872707486152649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:18:56,634 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for 169fe32fc4194690caf6575dbee1d411: 2024-11-08T06:18:56,636 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411., pid=29, masterSystemTime=1731046736607 2024-11-08T06:18:56,639 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:18:56,639 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:18:56,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=169fe32fc4194690caf6575dbee1d411, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:18:56,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T06:18:56,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure 169fe32fc4194690caf6575dbee1d411, server=d1f4bb6aa14f,36803,1731046638998 in 190 msec 2024-11-08T06:18:56,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-08T06:18:56,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=169fe32fc4194690caf6575dbee1d411, ASSIGN in 353 msec 2024-11-08T06:18:56,651 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:18:56,651 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046736651"}]},"ts":"1731046736651"} 2024-11-08T06:18:56,653 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:18:56,657 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:18:56,659 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 1.2030 sec 2024-11-08T06:18:57,769 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-08T06:18:57,770 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-08T06:18:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-08T06:18:58,472 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-08T06:18:58,472 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:18:58,472 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:58,479 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T06:18:58,479 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:18:58,479 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-08T06:18:58,483 DEBUG [hconnection-0x7a698487-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:58,486 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39174, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:58,503 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:18:58,506 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:18:58,507 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:18:58,514 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x5176280c {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:18:58,514 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x5176280c {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:18:58,517 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39178, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:18:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:18:58,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:18:58,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567348 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:58,523 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:18:58,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567348 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:18:58,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T06:18:58,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T06:18:58,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:58,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T06:18:58,773 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:18:58,773 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T06:18:58,774 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:01,529 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3005 2024-11-08T06:19:01,529 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58a43e93 to 127.0.0.1:49997 2024-11-08T06:19:01,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:01,530 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567348 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:19:01,530 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:01,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567348' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T06:19:01,530 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:01,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:39178 deadline: 1731046742529, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567348' does not exist 2024-11-08T06:19:01,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:01,530 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5176280c to 127.0.0.1:49997 2024-11-08T06:19:01,531 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:19:01,531 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d1f4bb6aa14f,36803,1731046638998 for closing scanner id = 8767934147168567348 for 169fe32fc4194690caf6575dbee1d411 of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T06:19:01,547 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=227 (was 225) - Thread LEAK? -, OpenFileDescriptor=454 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=492 (was 456) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3088 (was 3115) 2024-11-08T06:19:01,559 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=227, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=492, ProcessCount=11, AvailableMemoryMB=3088 2024-11-08T06:19:01,562 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x388157f0 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e77844a 2024-11-08T06:19:01,577 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x388157f0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13578f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:01,579 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a7ec048 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6676292d 2024-11-08T06:19:01,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547adcc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:01,588 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:01,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-08T06:19:01,592 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:01,592 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:01,592 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-08T06:19:01,593 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T06:19:01,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741851_1027 (size=417) 2024-11-08T06:19:01,647 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e60b79623b21cdaf896284976a0739d4, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:01,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741852_1028 (size=68) 2024-11-08T06:19:01,660 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:01,660 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing e60b79623b21cdaf896284976a0739d4, disabling compactions & flushes 2024-11-08T06:19:01,660 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:19:01,660 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:19:01,660 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. after waiting 0 ms 2024-11-08T06:19:01,660 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:19:01,660 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:19:01,660 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for e60b79623b21cdaf896284976a0739d4: 2024-11-08T06:19:01,662 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:01,662 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1731046741662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046741662"}]},"ts":"1731046741662"} 2024-11-08T06:19:01,667 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:01,668 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:01,668 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046741668"}]},"ts":"1731046741668"} 2024-11-08T06:19:01,671 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:19:01,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=e60b79623b21cdaf896284976a0739d4, ASSIGN}] 2024-11-08T06:19:01,677 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=e60b79623b21cdaf896284976a0739d4, ASSIGN 2024-11-08T06:19:01,678 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=e60b79623b21cdaf896284976a0739d4, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:01,829 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=e60b79623b21cdaf896284976a0739d4, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:01,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure e60b79623b21cdaf896284976a0739d4, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:01,985 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:01,991 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:19:01,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => e60b79623b21cdaf896284976a0739d4, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:01,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false e60b79623b21cdaf896284976a0739d4 2024-11-08T06:19:01,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:01,993 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for e60b79623b21cdaf896284976a0739d4 2024-11-08T06:19:01,993 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for e60b79623b21cdaf896284976a0739d4 2024-11-08T06:19:01,998 INFO [StoreOpener-e60b79623b21cdaf896284976a0739d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e60b79623b21cdaf896284976a0739d4 2024-11-08T06:19:02,000 INFO [StoreOpener-e60b79623b21cdaf896284976a0739d4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e60b79623b21cdaf896284976a0739d4 columnFamilyName testFamily 2024-11-08T06:19:02,000 DEBUG [StoreOpener-e60b79623b21cdaf896284976a0739d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:02,001 INFO [StoreOpener-e60b79623b21cdaf896284976a0739d4-1 {}] regionserver.HStore(327): Store=e60b79623b21cdaf896284976a0739d4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:02,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4 2024-11-08T06:19:02,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4 2024-11-08T06:19:02,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for e60b79623b21cdaf896284976a0739d4 2024-11-08T06:19:02,012 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:02,013 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened e60b79623b21cdaf896284976a0739d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71261234, jitterRate=0.06187513470649719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:02,013 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for e60b79623b21cdaf896284976a0739d4: 2024-11-08T06:19:02,015 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4., pid=32, masterSystemTime=1731046741985 2024-11-08T06:19:02,019 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:19:02,019 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:19:02,019 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=e60b79623b21cdaf896284976a0739d4, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:02,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T06:19:02,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure e60b79623b21cdaf896284976a0739d4, server=d1f4bb6aa14f,36803,1731046638998 in 190 msec 2024-11-08T06:19:02,025 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:19:02,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-08T06:19:02,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=e60b79623b21cdaf896284976a0739d4, ASSIGN in 350 msec 2024-11-08T06:19:02,029 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:02,029 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046742029"}]},"ts":"1731046742029"} 2024-11-08T06:19:02,031 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:19:02,035 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:02,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 447 msec 2024-11-08T06:19:02,117 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T06:19:02,601 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-08T06:19:02,601 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:19:02,601 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:02,616 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T06:19:02,616 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:02,617 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-08T06:19:02,621 DEBUG [hconnection-0x4feba97d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:02,633 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45288, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:02,648 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:02,652 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:19:02,655 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567352 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:19:02,655 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T06:19:03,771 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-08T06:19:03,772 INFO [master/d1f4bb6aa14f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T06:19:03,772 INFO [master/d1f4bb6aa14f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T06:19:07,494 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:08,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-08T06:19:08,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:09,156 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6503 2024-11-08T06:19:09,156 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5a7ec048 to 127.0.0.1:49997 2024-11-08T06:19:09,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:09,156 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:09,156 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:09,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:09,157 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x388157f0 to 127.0.0.1:49997 2024-11-08T06:19:09,170 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 227) - Thread LEAK? -, OpenFileDescriptor=452 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=501 (was 492) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3063 (was 3088) 2024-11-08T06:19:09,182 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=501, ProcessCount=11, AvailableMemoryMB=3062 2024-11-08T06:19:09,184 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4c2369d1 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a0b4f3e 2024-11-08T06:19:09,189 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x4c2369d1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d4b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:09,191 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05b3cf21 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6587f449 2024-11-08T06:19:09,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bd74c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:09,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-08T06:19:09,203 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:09,203 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:09,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-08T06:19:09,204 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T06:19:09,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741853_1029 (size=421) 2024-11-08T06:19:09,218 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 75cd2864f9d52d49cb12429ecdc3da83, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:09,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741854_1030 (size=72) 2024-11-08T06:19:09,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:09,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 75cd2864f9d52d49cb12429ecdc3da83, disabling compactions & flushes 2024-11-08T06:19:09,228 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:19:09,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:19:09,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. after waiting 0 ms 2024-11-08T06:19:09,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:19:09,228 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:19:09,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 75cd2864f9d52d49cb12429ecdc3da83: 2024-11-08T06:19:09,230 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:09,230 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731046749230"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046749230"}]},"ts":"1731046749230"} 2024-11-08T06:19:09,233 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:09,234 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:09,234 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046749234"}]},"ts":"1731046749234"} 2024-11-08T06:19:09,236 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:19:09,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=75cd2864f9d52d49cb12429ecdc3da83, ASSIGN}] 2024-11-08T06:19:09,242 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=75cd2864f9d52d49cb12429ecdc3da83, ASSIGN 2024-11-08T06:19:09,244 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=75cd2864f9d52d49cb12429ecdc3da83, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:09,394 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=75cd2864f9d52d49cb12429ecdc3da83, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:09,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure 75cd2864f9d52d49cb12429ecdc3da83, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:09,551 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:09,557 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:19:09,558 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => 75cd2864f9d52d49cb12429ecdc3da83, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:09,558 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false 75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:19:09,558 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:09,559 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for 75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:19:09,559 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for 75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:19:09,561 INFO [StoreOpener-75cd2864f9d52d49cb12429ecdc3da83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:19:09,563 INFO [StoreOpener-75cd2864f9d52d49cb12429ecdc3da83-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75cd2864f9d52d49cb12429ecdc3da83 columnFamilyName testFamily 2024-11-08T06:19:09,564 DEBUG [StoreOpener-75cd2864f9d52d49cb12429ecdc3da83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:09,564 INFO [StoreOpener-75cd2864f9d52d49cb12429ecdc3da83-1 {}] regionserver.HStore(327): Store=75cd2864f9d52d49cb12429ecdc3da83/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:09,566 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:19:09,566 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:19:09,569 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for 75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:19:09,573 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:09,574 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened 75cd2864f9d52d49cb12429ecdc3da83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61948832, jitterRate=-0.07689046859741211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:09,574 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for 75cd2864f9d52d49cb12429ecdc3da83: 2024-11-08T06:19:09,576 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83., pid=35, masterSystemTime=1731046749551 2024-11-08T06:19:09,578 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:19:09,578 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:19:09,579 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=75cd2864f9d52d49cb12429ecdc3da83, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:09,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-08T06:19:09,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure 75cd2864f9d52d49cb12429ecdc3da83, server=d1f4bb6aa14f,36803,1731046638998 in 184 msec 2024-11-08T06:19:09,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T06:19:09,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=75cd2864f9d52d49cb12429ecdc3da83, ASSIGN in 345 msec 2024-11-08T06:19:09,589 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:09,589 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046749589"}]},"ts":"1731046749589"} 2024-11-08T06:19:09,592 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:19:09,595 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:09,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 396 msec 2024-11-08T06:19:09,773 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-08T06:19:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-08T06:19:10,208 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-08T06:19:10,208 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:19:10,208 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:10,214 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-08T06:19:10,214 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:10,214 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-08T06:19:10,217 DEBUG [hconnection-0x2a2301ef-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:10,219 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40518, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:10,231 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:10,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:19:10,250 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567356 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:19:10,251 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T06:19:11,752 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1513 2024-11-08T06:19:11,752 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x05b3cf21 to 127.0.0.1:49997 2024-11-08T06:19:11,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:11,752 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:11,753 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:11,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:11,753 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4c2369d1 to 127.0.0.1:49997 2024-11-08T06:19:11,765 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=454 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=509 (was 501) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3068 (was 3062) - AvailableMemoryMB LEAK? - 2024-11-08T06:19:11,776 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=509, ProcessCount=11, AvailableMemoryMB=3066 2024-11-08T06:19:11,779 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f702943 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26f56cd 2024-11-08T06:19:11,784 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x6f702943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:11,786 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cacc2c2 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@40d2969b 2024-11-08T06:19:11,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0695f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:11,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-08T06:19:11,796 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:11,796 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:11,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-08T06:19:11,797 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T06:19:11,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741855_1031 (size=420) 2024-11-08T06:19:11,818 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 506c2ac65720fba05f47cd2e4def3562, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:11,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741856_1032 (size=71) 2024-11-08T06:19:12,269 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:12,269 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 506c2ac65720fba05f47cd2e4def3562, disabling compactions & flushes 2024-11-08T06:19:12,269 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:19:12,269 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:19:12,269 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. after waiting 0 ms 2024-11-08T06:19:12,269 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:19:12,269 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:19:12,269 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 506c2ac65720fba05f47cd2e4def3562: 2024-11-08T06:19:12,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:12,271 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731046752271"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046752271"}]},"ts":"1731046752271"} 2024-11-08T06:19:12,274 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:12,275 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:12,275 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046752275"}]},"ts":"1731046752275"} 2024-11-08T06:19:12,277 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:19:12,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=506c2ac65720fba05f47cd2e4def3562, ASSIGN}] 2024-11-08T06:19:12,286 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=506c2ac65720fba05f47cd2e4def3562, ASSIGN 2024-11-08T06:19:12,287 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=506c2ac65720fba05f47cd2e4def3562, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:12,437 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=506c2ac65720fba05f47cd2e4def3562, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:12,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 506c2ac65720fba05f47cd2e4def3562, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:12,593 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:12,604 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:19:12,604 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 506c2ac65720fba05f47cd2e4def3562, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:12,604 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:19:12,605 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:12,605 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:19:12,605 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:19:12,609 INFO [StoreOpener-506c2ac65720fba05f47cd2e4def3562-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:19:12,612 INFO [StoreOpener-506c2ac65720fba05f47cd2e4def3562-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c2ac65720fba05f47cd2e4def3562 columnFamilyName testFamily 2024-11-08T06:19:12,612 DEBUG [StoreOpener-506c2ac65720fba05f47cd2e4def3562-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:12,612 INFO [StoreOpener-506c2ac65720fba05f47cd2e4def3562-1 {}] regionserver.HStore(327): Store=506c2ac65720fba05f47cd2e4def3562/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:12,614 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:19:12,615 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:19:12,619 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:19:12,628 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:12,629 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 506c2ac65720fba05f47cd2e4def3562; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68481645, jitterRate=0.02045603096485138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:12,630 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 506c2ac65720fba05f47cd2e4def3562: 2024-11-08T06:19:12,632 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562., pid=38, masterSystemTime=1731046752593 2024-11-08T06:19:12,635 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:19:12,635 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:19:12,636 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=506c2ac65720fba05f47cd2e4def3562, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:12,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-08T06:19:12,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 506c2ac65720fba05f47cd2e4def3562, server=d1f4bb6aa14f,36803,1731046638998 in 197 msec 2024-11-08T06:19:12,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T06:19:12,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=506c2ac65720fba05f47cd2e4def3562, ASSIGN in 356 msec 2024-11-08T06:19:12,644 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:12,645 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046752644"}]},"ts":"1731046752644"} 2024-11-08T06:19:12,647 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:19:12,650 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:12,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 857 msec 2024-11-08T06:19:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-08T06:19:12,801 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-08T06:19:12,801 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:19:12,801 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:12,807 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T06:19:12,807 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:12,807 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-08T06:19:12,810 DEBUG [hconnection-0x72f683b9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:12,812 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40530, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:12,824 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:12,830 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x476c1276 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@63a41208 2024-11-08T06:19:12,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb755ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:12,835 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:19:12,838 DEBUG [hconnection-0x6f06d652-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:12,842 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40546, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:12,847 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:19:12,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567361 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:12,850 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:19:12,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567361 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:12,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T06:19:13,859 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T06:19:13.859Z, RpcRetryingCaller{globalStartTime=2024-11-08T06:19:12.850Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T06:19:13.859Z, RpcRetryingCaller{globalStartTime=2024-11-08T06:19:12.850Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:19:13,859 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1009 2024-11-08T06:19:13,859 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x476c1276 to 127.0.0.1:49997 2024-11-08T06:19:13,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:13,860 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0cacc2c2 to 127.0.0.1:49997 2024-11-08T06:19:13,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:13,860 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:13,861 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:13,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:13,861 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f702943 to 127.0.0.1:49997 2024-11-08T06:19:13,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 231) - Thread LEAK? -, OpenFileDescriptor=459 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=509 (was 509), ProcessCount=11 (was 11), AvailableMemoryMB=3051 (was 3066) 2024-11-08T06:19:13,882 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=509, ProcessCount=11, AvailableMemoryMB=3050 2024-11-08T06:19:13,885 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d053001 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@238f575b 2024-11-08T06:19:13,890 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x2d053001 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f8217b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:13,892 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x229c45cb to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@128c8246 2024-11-08T06:19:13,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160d2d2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:13,902 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:13,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T06:19:13,905 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:13,905 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:13,906 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-08T06:19:13,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T06:19:13,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741857_1033 (size=422) 2024-11-08T06:19:13,918 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 58b3ddfd049d1caa4403adb146662c75, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:13,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741858_1034 (size=73) 2024-11-08T06:19:14,328 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:14,328 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 58b3ddfd049d1caa4403adb146662c75, disabling compactions & flushes 2024-11-08T06:19:14,328 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:19:14,328 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:19:14,328 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. after waiting 0 ms 2024-11-08T06:19:14,328 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:19:14,328 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:19:14,328 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 58b3ddfd049d1caa4403adb146662c75: 2024-11-08T06:19:14,330 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:14,330 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731046754330"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046754330"}]},"ts":"1731046754330"} 2024-11-08T06:19:14,333 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:14,334 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:14,334 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046754334"}]},"ts":"1731046754334"} 2024-11-08T06:19:14,336 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:19:14,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=58b3ddfd049d1caa4403adb146662c75, ASSIGN}] 2024-11-08T06:19:14,341 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=58b3ddfd049d1caa4403adb146662c75, ASSIGN 2024-11-08T06:19:14,342 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=58b3ddfd049d1caa4403adb146662c75, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:14,351 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:19:14,493 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=58b3ddfd049d1caa4403adb146662c75, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:14,496 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 58b3ddfd049d1caa4403adb146662c75, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:14,648 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:14,654 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:19:14,654 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 58b3ddfd049d1caa4403adb146662c75, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:14,655 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:19:14,655 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:14,655 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:19:14,655 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:19:14,662 INFO [StoreOpener-58b3ddfd049d1caa4403adb146662c75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:19:14,664 INFO [StoreOpener-58b3ddfd049d1caa4403adb146662c75-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58b3ddfd049d1caa4403adb146662c75 columnFamilyName testFamily 2024-11-08T06:19:14,664 DEBUG [StoreOpener-58b3ddfd049d1caa4403adb146662c75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:14,665 INFO [StoreOpener-58b3ddfd049d1caa4403adb146662c75-1 {}] regionserver.HStore(327): Store=58b3ddfd049d1caa4403adb146662c75/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:14,666 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:19:14,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:19:14,671 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:19:14,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:14,676 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 58b3ddfd049d1caa4403adb146662c75; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66436614, jitterRate=-0.010017305612564087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:14,677 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 58b3ddfd049d1caa4403adb146662c75: 2024-11-08T06:19:14,678 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75., pid=41, masterSystemTime=1731046754648 2024-11-08T06:19:14,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:19:14,681 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:19:14,682 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=58b3ddfd049d1caa4403adb146662c75, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:14,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T06:19:14,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 58b3ddfd049d1caa4403adb146662c75, server=d1f4bb6aa14f,36803,1731046638998 in 190 msec 2024-11-08T06:19:14,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-08T06:19:14,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=58b3ddfd049d1caa4403adb146662c75, ASSIGN in 348 msec 2024-11-08T06:19:14,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:14,697 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046754697"}]},"ts":"1731046754697"} 2024-11-08T06:19:14,699 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:19:14,703 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:14,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 800 msec 2024-11-08T06:19:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-08T06:19:14,911 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-08T06:19:14,911 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:19:14,911 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:14,917 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-08T06:19:14,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:14,918 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-08T06:19:14,920 DEBUG [hconnection-0x660384b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:14,922 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40556, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:14,933 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:14,936 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:19:14,940 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:19:14,940 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T06:19:15,060 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:15,775 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-08T06:19:15,775 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-08T06:19:17,402 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:19:18,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T06:19:18,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:18,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T06:19:18,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:18,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-08T06:19:18,773 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:20,156 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T06:19:21,960 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:19:21,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40556 deadline: 1731046763939, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:19:21,961 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:19:21,962 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567365 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:19:21,962 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7025 2024-11-08T06:19:21,962 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x229c45cb to 127.0.0.1:49997 2024-11-08T06:19:21,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:21,963 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:21,963 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:21,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:21,963 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d053001 to 127.0.0.1:49997 2024-11-08T06:19:21,973 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=230 (was 236), OpenFileDescriptor=454 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=455 (was 509), ProcessCount=11 (was 11), AvailableMemoryMB=3037 (was 3050) 2024-11-08T06:19:21,983 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=230, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=455, ProcessCount=11, AvailableMemoryMB=3037 2024-11-08T06:19:21,986 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x762af7e8 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7cef7242 2024-11-08T06:19:21,989 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x762af7e8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58947443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:21,991 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x523c9018 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6cba182f 2024-11-08T06:19:21,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0e27e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:21,998 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:21,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T06:19:22,000 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:22,000 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:22,000 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-08T06:19:22,001 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T06:19:22,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741859_1035 (size=428) 2024-11-08T06:19:22,149 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:45761,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:19:22,411 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 13de1ea4d8dd655db7256497e2b35cd6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:22,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741860_1036 (size=79) 2024-11-08T06:19:22,419 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:22,419 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 13de1ea4d8dd655db7256497e2b35cd6, disabling compactions & flushes 2024-11-08T06:19:22,419 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:19:22,419 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:19:22,419 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. after waiting 0 ms 2024-11-08T06:19:22,419 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:19:22,420 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:19:22,420 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 13de1ea4d8dd655db7256497e2b35cd6: 2024-11-08T06:19:22,421 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:22,421 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731046762421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046762421"}]},"ts":"1731046762421"} 2024-11-08T06:19:22,423 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:22,425 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:22,425 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046762425"}]},"ts":"1731046762425"} 2024-11-08T06:19:22,427 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:19:22,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=13de1ea4d8dd655db7256497e2b35cd6, ASSIGN}] 2024-11-08T06:19:22,432 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=13de1ea4d8dd655db7256497e2b35cd6, ASSIGN 2024-11-08T06:19:22,432 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=13de1ea4d8dd655db7256497e2b35cd6, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:22,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=13de1ea4d8dd655db7256497e2b35cd6, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:22,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 13de1ea4d8dd655db7256497e2b35cd6, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:22,738 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:22,743 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:19:22,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 13de1ea4d8dd655db7256497e2b35cd6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:22,744 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:19:22,744 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:22,744 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:19:22,744 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:19:22,746 INFO [StoreOpener-13de1ea4d8dd655db7256497e2b35cd6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:19:22,748 INFO [StoreOpener-13de1ea4d8dd655db7256497e2b35cd6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13de1ea4d8dd655db7256497e2b35cd6 columnFamilyName testFamily 2024-11-08T06:19:22,748 DEBUG [StoreOpener-13de1ea4d8dd655db7256497e2b35cd6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:22,749 INFO [StoreOpener-13de1ea4d8dd655db7256497e2b35cd6-1 {}] regionserver.HStore(327): Store=13de1ea4d8dd655db7256497e2b35cd6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:22,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:19:22,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:19:22,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:19:22,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:22,757 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened 13de1ea4d8dd655db7256497e2b35cd6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71562368, jitterRate=0.06636238098144531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:22,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 13de1ea4d8dd655db7256497e2b35cd6: 2024-11-08T06:19:22,759 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., pid=44, masterSystemTime=1731046762738 2024-11-08T06:19:22,761 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:19:22,761 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:19:22,761 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=13de1ea4d8dd655db7256497e2b35cd6, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:22,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-08T06:19:22,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 13de1ea4d8dd655db7256497e2b35cd6, server=d1f4bb6aa14f,36803,1731046638998 in 178 msec 2024-11-08T06:19:22,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T06:19:22,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=13de1ea4d8dd655db7256497e2b35cd6, ASSIGN in 335 msec 2024-11-08T06:19:22,769 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:22,769 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046762769"}]},"ts":"1731046762769"} 2024-11-08T06:19:22,771 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:19:22,774 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:22,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 776 msec 2024-11-08T06:19:22,826 DEBUG [master/d1f4bb6aa14f:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T06:19:22,831 DEBUG [master/d1f4bb6aa14f:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 640f50ffb597620ddc4800f241b7ca8f changed from -1.0 to 0.0, refreshing cache 2024-11-08T06:19:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-08T06:19:23,004 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-08T06:19:23,004 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:19:23,004 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:23,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T06:19:23,008 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:23,008 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T06:19:23,011 DEBUG [hconnection-0x1d771c6e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:23,012 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53420, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:23,021 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:23,024 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:19:23,025 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:19:23,026 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x762af7e8 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:19:23,026 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x762af7e8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:23,027 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53426, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:19:23,030 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:19:25,463 ERROR [LeaseRenewer:jenkins@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:45761,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:19:28,245 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:28,282 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T06:19:28,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T06:19:28,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:29,039 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-08T06:19:29,040 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-08T06:19:29,040 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:19:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T06:19:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:19:30,057 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:19:32,029 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1968ms 2024-11-08T06:19:32,029 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1968ms 2024-11-08T06:19:32,029 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:19:32,030 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T06:19:29.040Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2024-11-08T06:19:32.029Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1968ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1968ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T06:19:32,032 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9007 2024-11-08T06:19:32,032 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x523c9018 to 127.0.0.1:49997 2024-11-08T06:19:32,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:32,032 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:32,032 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:32,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:32,033 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x762af7e8 to 127.0.0.1:49997 2024-11-08T06:19:32,043 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=229 (was 230), OpenFileDescriptor=448 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=422 (was 455), ProcessCount=11 (was 11), AvailableMemoryMB=3906 (was 3037) - AvailableMemoryMB LEAK? - 2024-11-08T06:19:32,051 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=422, ProcessCount=11, AvailableMemoryMB=3905 2024-11-08T06:19:32,053 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d9c252d to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70c036d9 2024-11-08T06:19:32,056 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x7d9c252d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158c31c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:32,057 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e65352b to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79458f02 2024-11-08T06:19:32,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656c684a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:32,063 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:32,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-08T06:19:32,065 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:32,065 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:32,065 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-08T06:19:32,066 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T06:19:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741861_1037 (size=416) 2024-11-08T06:19:32,477 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 79c06b38371ce93f62765ff1ce21f968, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:32,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741862_1038 (size=67) 2024-11-08T06:19:32,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:32,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 79c06b38371ce93f62765ff1ce21f968, disabling compactions & flushes 2024-11-08T06:19:32,886 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:19:32,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:19:32,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. after waiting 0 ms 2024-11-08T06:19:32,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:19:32,886 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:19:32,886 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 79c06b38371ce93f62765ff1ce21f968: 2024-11-08T06:19:32,888 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:32,888 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731046772888"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046772888"}]},"ts":"1731046772888"} 2024-11-08T06:19:32,891 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:32,892 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:32,892 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046772892"}]},"ts":"1731046772892"} 2024-11-08T06:19:32,895 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:19:32,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=79c06b38371ce93f62765ff1ce21f968, ASSIGN}] 2024-11-08T06:19:32,904 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=79c06b38371ce93f62765ff1ce21f968, ASSIGN 2024-11-08T06:19:32,905 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=79c06b38371ce93f62765ff1ce21f968, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:33,056 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=79c06b38371ce93f62765ff1ce21f968, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:33,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 79c06b38371ce93f62765ff1ce21f968, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T06:19:33,211 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:33,217 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:19:33,218 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 79c06b38371ce93f62765ff1ce21f968, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:33,218 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:19:33,218 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:33,218 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:19:33,218 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:19:33,221 INFO [StoreOpener-79c06b38371ce93f62765ff1ce21f968-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:19:33,223 INFO [StoreOpener-79c06b38371ce93f62765ff1ce21f968-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79c06b38371ce93f62765ff1ce21f968 columnFamilyName testFamily 2024-11-08T06:19:33,223 DEBUG [StoreOpener-79c06b38371ce93f62765ff1ce21f968-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:33,223 INFO [StoreOpener-79c06b38371ce93f62765ff1ce21f968-1 {}] regionserver.HStore(327): Store=79c06b38371ce93f62765ff1ce21f968/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:33,225 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:19:33,225 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:19:33,228 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:19:33,232 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:33,233 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 79c06b38371ce93f62765ff1ce21f968; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71714101, jitterRate=0.0686233788728714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:33,234 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 79c06b38371ce93f62765ff1ce21f968: 2024-11-08T06:19:33,235 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968., pid=47, masterSystemTime=1731046773211 2024-11-08T06:19:33,237 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:19:33,237 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:19:33,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=79c06b38371ce93f62765ff1ce21f968, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:33,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T06:19:33,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 79c06b38371ce93f62765ff1ce21f968, server=d1f4bb6aa14f,36803,1731046638998 in 181 msec 2024-11-08T06:19:33,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-08T06:19:33,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=79c06b38371ce93f62765ff1ce21f968, ASSIGN in 339 msec 2024-11-08T06:19:33,244 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:33,245 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046773244"}]},"ts":"1731046773244"} 2024-11-08T06:19:33,246 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:19:33,249 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:33,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.1860 sec 2024-11-08T06:19:34,283 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-08T06:19:34,625 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567316 expired clientIPAndPort=172.17.0.2:39508, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:19:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T06:19:35,071 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-08T06:19:35,071 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:19:35,071 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:35,075 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-08T06:19:35,075 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:35,075 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-08T06:19:35,077 DEBUG [hconnection-0xa0e159-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:35,079 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33100, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:35,087 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:35,091 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x45976e97 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56f1de71 2024-11-08T06:19:35,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244998cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:35,095 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:19:35,097 DEBUG [hconnection-0x7c93c861-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:35,098 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33110, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:35,100 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:19:35,101 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567378 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:35,102 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:19:35,102 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567378 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:35,102 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T06:19:36,557 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:19:38,720 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-08T06:19:38,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:41,109 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T06:19:41.108Z, RpcRetryingCaller{globalStartTime=2024-11-08T06:19:35.102Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-08T06:19:41.108Z, RpcRetryingCaller{globalStartTime=2024-11-08T06:19:35.102Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:19:41,109 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6007 2024-11-08T06:19:41,109 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x45976e97 to 127.0.0.1:49997 2024-11-08T06:19:41,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:41,109 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e65352b to 127.0.0.1:49997 2024-11-08T06:19:41,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:41,109 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:41,110 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:41,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:41,110 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d9c252d to 127.0.0.1:49997 2024-11-08T06:19:41,120 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=233 (was 229) - Thread LEAK? -, OpenFileDescriptor=451 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 422), ProcessCount=11 (was 11), AvailableMemoryMB=3806 (was 3905) 2024-11-08T06:19:41,128 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=233, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=404, ProcessCount=11, AvailableMemoryMB=3806 2024-11-08T06:19:41,130 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1223e70b to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@480e4995 2024-11-08T06:19:41,133 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1223e70b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1400327b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:41,134 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a6f8b19 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e0d59dd 2024-11-08T06:19:41,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bfca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:41,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T06:19:41,143 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:41,143 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:41,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-08T06:19:41,144 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T06:19:41,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741863_1039 (size=432) 2024-11-08T06:19:41,559 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1052053f1be9a398e9e6217d04d32d31, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:41,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741864_1040 (size=83) 2024-11-08T06:19:41,603 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:19:41,631 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567318 expired clientIPAndPort=172.17.0.2:39508, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:19:41,967 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:41,967 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 1052053f1be9a398e9e6217d04d32d31, disabling compactions & flushes 2024-11-08T06:19:41,967 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:19:41,967 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:19:41,967 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. after waiting 0 ms 2024-11-08T06:19:41,967 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:19:41,967 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:19:41,967 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1052053f1be9a398e9e6217d04d32d31: 2024-11-08T06:19:41,969 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:41,969 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731046781969"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046781969"}]},"ts":"1731046781969"} 2024-11-08T06:19:41,971 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:41,972 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:41,973 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046781972"}]},"ts":"1731046781972"} 2024-11-08T06:19:41,974 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-08T06:19:41,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=1052053f1be9a398e9e6217d04d32d31, ASSIGN}] 2024-11-08T06:19:41,980 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=1052053f1be9a398e9e6217d04d32d31, ASSIGN 2024-11-08T06:19:41,980 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=1052053f1be9a398e9e6217d04d32d31, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:42,131 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=1052053f1be9a398e9e6217d04d32d31, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:42,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure 1052053f1be9a398e9e6217d04d32d31, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T06:19:42,285 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:42,290 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:19:42,291 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => 1052053f1be9a398e9e6217d04d32d31, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:42,291 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false 1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:19:42,291 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:42,291 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for 1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:19:42,291 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for 1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:19:42,293 INFO [StoreOpener-1052053f1be9a398e9e6217d04d32d31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:19:42,295 INFO [StoreOpener-1052053f1be9a398e9e6217d04d32d31-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1052053f1be9a398e9e6217d04d32d31 columnFamilyName testFamily 2024-11-08T06:19:42,295 DEBUG [StoreOpener-1052053f1be9a398e9e6217d04d32d31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:42,295 INFO [StoreOpener-1052053f1be9a398e9e6217d04d32d31-1 {}] regionserver.HStore(327): Store=1052053f1be9a398e9e6217d04d32d31/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:42,297 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:19:42,297 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:19:42,300 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for 1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:19:42,303 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:42,303 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened 1052053f1be9a398e9e6217d04d32d31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74620961, jitterRate=0.1119389683008194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:42,305 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for 1052053f1be9a398e9e6217d04d32d31: 2024-11-08T06:19:42,306 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., pid=50, masterSystemTime=1731046782285 2024-11-08T06:19:42,308 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:19:42,308 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:19:42,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=1052053f1be9a398e9e6217d04d32d31, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:42,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-08T06:19:42,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure 1052053f1be9a398e9e6217d04d32d31, server=d1f4bb6aa14f,36803,1731046638998 in 177 msec 2024-11-08T06:19:42,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T06:19:42,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=1052053f1be9a398e9e6217d04d32d31, ASSIGN in 334 msec 2024-11-08T06:19:42,315 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:42,315 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046782315"}]},"ts":"1731046782315"} 2024-11-08T06:19:42,316 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-08T06:19:42,319 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:42,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.1780 sec 2024-11-08T06:19:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-08T06:19:44,149 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-08T06:19:44,149 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-08T06:19:44,149 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:44,153 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-08T06:19:44,153 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:44,153 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-08T06:19:44,156 DEBUG [hconnection-0x57878035-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:44,158 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51494, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:44,167 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:44,169 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:19:44,170 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:19:44,171 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1223e70b {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:19:44,171 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1223e70b {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:44,172 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51498, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:19:44,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:19:45,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-08T06:19:45,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-08T06:19:45,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:19:45,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T06:19:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:19:46,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:19:46,285 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-08T06:19:47,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=993ms, rpcTimeout=985ms 2024-11-08T06:19:47,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=993ms, rpcTimeout=985ms 2024-11-08T06:19:47,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:19:47,179 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T06:19:45.179Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-08T06:19:47.179Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=993ms, rpcTimeout=985ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=993ms, rpcTimeout=985ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=993ms, rpcTimeout=985ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T06:19:47,180 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3011 2024-11-08T06:19:47,181 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2a6f8b19 to 127.0.0.1:49997 2024-11-08T06:19:47,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:47,181 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:47,181 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:47,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:47,181 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1223e70b to 127.0.0.1:49997 2024-11-08T06:19:47,190 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=230 (was 233), OpenFileDescriptor=448 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=349 (was 404), ProcessCount=11 (was 11), AvailableMemoryMB=3781 (was 3806) 2024-11-08T06:19:47,199 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=349, ProcessCount=11, AvailableMemoryMB=3781 2024-11-08T06:19:47,201 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d1bb60c to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d2a852 2024-11-08T06:19:47,204 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x7d1bb60c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f43f7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:47,205 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32b3cd9e to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a47aa81 2024-11-08T06:19:47,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4cc413, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:47,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T06:19:47,214 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:47,214 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:47,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-08T06:19:47,215 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T06:19:47,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741865_1041 (size=436) 2024-11-08T06:19:47,227 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e8e8dada2468fd2a22641e1cf25b4ac0, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:47,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741866_1042 (size=87) 2024-11-08T06:19:47,402 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:19:47,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:47,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing e8e8dada2468fd2a22641e1cf25b4ac0, disabling compactions & flushes 2024-11-08T06:19:47,636 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:19:47,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:19:47,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. after waiting 0 ms 2024-11-08T06:19:47,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:19:47,636 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:19:47,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for e8e8dada2468fd2a22641e1cf25b4ac0: 2024-11-08T06:19:47,637 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:47,638 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1731046787637"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046787637"}]},"ts":"1731046787637"} 2024-11-08T06:19:47,640 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:47,640 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:47,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046787640"}]},"ts":"1731046787640"} 2024-11-08T06:19:47,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:19:47,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=e8e8dada2468fd2a22641e1cf25b4ac0, ASSIGN}] 2024-11-08T06:19:47,653 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=e8e8dada2468fd2a22641e1cf25b4ac0, ASSIGN 2024-11-08T06:19:47,654 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=e8e8dada2468fd2a22641e1cf25b4ac0, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:47,686 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:19:47,736 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567323 expired clientIPAndPort=172.17.0.2:58654, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:19:47,793 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:47,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=e8e8dada2468fd2a22641e1cf25b4ac0, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:47,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure e8e8dada2468fd2a22641e1cf25b4ac0, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:47,959 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:47,964 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:19:47,964 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => e8e8dada2468fd2a22641e1cf25b4ac0, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:47,965 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:19:47,965 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:47,965 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:19:47,965 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:19:47,967 INFO [StoreOpener-e8e8dada2468fd2a22641e1cf25b4ac0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:19:47,968 INFO [StoreOpener-e8e8dada2468fd2a22641e1cf25b4ac0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e8dada2468fd2a22641e1cf25b4ac0 columnFamilyName testFamily 2024-11-08T06:19:47,968 DEBUG [StoreOpener-e8e8dada2468fd2a22641e1cf25b4ac0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:47,969 INFO [StoreOpener-e8e8dada2468fd2a22641e1cf25b4ac0-1 {}] regionserver.HStore(327): Store=e8e8dada2468fd2a22641e1cf25b4ac0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:47,970 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:19:47,970 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:19:47,972 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:19:47,975 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:47,975 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened e8e8dada2468fd2a22641e1cf25b4ac0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75122899, jitterRate=0.1194184273481369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:47,976 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for e8e8dada2468fd2a22641e1cf25b4ac0: 2024-11-08T06:19:47,977 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0., pid=53, masterSystemTime=1731046787959 2024-11-08T06:19:47,979 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:19:47,979 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:19:47,979 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=e8e8dada2468fd2a22641e1cf25b4ac0, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:47,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T06:19:47,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure e8e8dada2468fd2a22641e1cf25b4ac0, server=d1f4bb6aa14f,36803,1731046638998 in 174 msec 2024-11-08T06:19:47,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-08T06:19:47,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=e8e8dada2468fd2a22641e1cf25b4ac0, ASSIGN in 331 msec 2024-11-08T06:19:47,985 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:47,986 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046787985"}]},"ts":"1731046787985"} 2024-11-08T06:19:47,987 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:19:47,990 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:47,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 779 msec 2024-11-08T06:19:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T06:19:48,217 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-08T06:19:48,217 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:19:48,218 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:48,222 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T06:19:48,222 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:48,223 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-08T06:19:48,225 DEBUG [hconnection-0x7c3a1970-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:48,226 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51506, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:48,234 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:48,236 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T06:19:48,237 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:19:48,238 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x7d1bb60c {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:19:48,238 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x7d1bb60c {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:48,239 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51510, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:19:48,241 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T06:19:48,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567390 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:48,242 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T06:19:48,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567390 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:48,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567390 2024-11-08T06:19:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51510 deadline: 1731046791241, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:19:48,244 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567390 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:19:48,244 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T06:19:48,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567391 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:48,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T06:19:48,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:19:48,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:19:48,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567392 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:48,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567392 2024-11-08T06:19:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51510 deadline: 1731046791245, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:19:48,247 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567392 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:19:48,248 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:19:48,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567393 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:48,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=8767934147168567393 2024-11-08T06:19:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51510 deadline: 1731046791247, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:19:48,249 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567393 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:19:48,250 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:19:48,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567394 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:19:48,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=8767934147168567394 2024-11-08T06:19:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51510 deadline: 1731046791249, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:19:48,251 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567394 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:19:48,252 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:19:48,252 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32b3cd9e to 127.0.0.1:49997 2024-11-08T06:19:48,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:48,252 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:19:48,253 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:19:48,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:19:48,253 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d1bb60c to 127.0.0.1:49997 2024-11-08T06:19:48,263 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=232 (was 230) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=349 (was 349), ProcessCount=11 (was 11), AvailableMemoryMB=3753 (was 3781) 2024-11-08T06:19:48,273 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=232, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=349, ProcessCount=11, AvailableMemoryMB=3750 2024-11-08T06:19:48,275 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46e87ba6 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1751b6d2 2024-11-08T06:19:48,278 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x46e87ba6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5604f403, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:48,279 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5cb3e1e9 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@381140ba 2024-11-08T06:19:48,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d213955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:19:48,285 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:19:48,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T06:19:48,287 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:19:48,287 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:48,287 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-08T06:19:48,287 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:19:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T06:19:48,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741867_1043 (size=422) 2024-11-08T06:19:48,337 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567330 expired clientIPAndPort=172.17.0.2:40310, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:19:48,337 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567329 expired clientIPAndPort=172.17.0.2:40310, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:19:48,337 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567327 expired clientIPAndPort=172.17.0.2:40310, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:19:48,698 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 40465c665e78ed129aec58b4cd8a7ce6, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:19:48,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741868_1044 (size=73) 2024-11-08T06:19:48,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T06:19:48,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:48,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T06:19:48,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-08T06:19:49,106 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:49,106 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 40465c665e78ed129aec58b4cd8a7ce6, disabling compactions & flushes 2024-11-08T06:19:49,106 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:19:49,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:19:49,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. after waiting 0 ms 2024-11-08T06:19:49,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:19:49,107 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:19:49,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 40465c665e78ed129aec58b4cd8a7ce6: 2024-11-08T06:19:49,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:19:49,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731046789108"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046789108"}]},"ts":"1731046789108"} 2024-11-08T06:19:49,110 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:19:49,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:19:49,111 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046789111"}]},"ts":"1731046789111"} 2024-11-08T06:19:49,113 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:19:49,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=40465c665e78ed129aec58b4cd8a7ce6, ASSIGN}] 2024-11-08T06:19:49,118 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=40465c665e78ed129aec58b4cd8a7ce6, ASSIGN 2024-11-08T06:19:49,119 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=40465c665e78ed129aec58b4cd8a7ce6, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:19:49,269 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=40465c665e78ed129aec58b4cd8a7ce6, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:49,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure 40465c665e78ed129aec58b4cd8a7ce6, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:19:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T06:19:49,424 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:49,429 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:19:49,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => 40465c665e78ed129aec58b4cd8a7ce6, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:19:49,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true 40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:19:49,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:19:49,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for 40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:19:49,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for 40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:19:49,431 INFO [StoreOpener-40465c665e78ed129aec58b4cd8a7ce6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:19:49,433 INFO [StoreOpener-40465c665e78ed129aec58b4cd8a7ce6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40465c665e78ed129aec58b4cd8a7ce6 columnFamilyName testFamily 2024-11-08T06:19:49,433 DEBUG [StoreOpener-40465c665e78ed129aec58b4cd8a7ce6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:19:49,433 INFO [StoreOpener-40465c665e78ed129aec58b4cd8a7ce6-1 {}] regionserver.HStore(327): Store=40465c665e78ed129aec58b4cd8a7ce6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:19:49,434 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:19:49,435 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:19:49,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for 40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:19:49,440 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:19:49,440 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened 40465c665e78ed129aec58b4cd8a7ce6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71057822, jitterRate=0.05884405970573425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:19:49,442 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for 40465c665e78ed129aec58b4cd8a7ce6: 2024-11-08T06:19:49,443 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6., pid=56, masterSystemTime=1731046789424 2024-11-08T06:19:49,445 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:19:49,445 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:19:49,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=40465c665e78ed129aec58b4cd8a7ce6, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:19:49,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T06:19:49,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure 40465c665e78ed129aec58b4cd8a7ce6, server=d1f4bb6aa14f,36803,1731046638998 in 175 msec 2024-11-08T06:19:49,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-08T06:19:49,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=40465c665e78ed129aec58b4cd8a7ce6, ASSIGN in 332 msec 2024-11-08T06:19:49,451 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:19:49,451 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046789451"}]},"ts":"1731046789451"} 2024-11-08T06:19:49,453 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:19:49,456 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:19:49,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1710 sec 2024-11-08T06:19:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-08T06:19:51,293 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-08T06:19:51,293 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:19:51,293 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:51,297 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T06:19:51,297 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:19:51,297 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T06:19:51,299 DEBUG [hconnection-0x625a0604-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:19:51,301 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41540, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:19:51,309 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:19:51,311 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:19:51,313 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:19:51,439 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567336 expired clientIPAndPort=172.17.0.2:40320, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:19:52,287 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-08T06:19:52,287 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-08T06:19:52,758 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:45761,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:19:53,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:19:53,942 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567341 expired clientIPAndPort=172.17.0.2:39168, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:19:55,943 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567343 expired clientIPAndPort=172.17.0.2:39168, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:19:57,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T06:19:58,330 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:19:58,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T06:19:58,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:00,319 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:20:00,319 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T06:20:00.319Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11017: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1989ms, rpcTimeout=1983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:20:00,320 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9008 2024-11-08T06:20:00,321 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5cb3e1e9 to 127.0.0.1:49997 2024-11-08T06:20:00,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:00,321 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:00,321 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:00,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:00,321 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x46e87ba6 to 127.0.0.1:49997 2024-11-08T06:20:00,330 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=227 (was 232), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=326 (was 349), ProcessCount=11 (was 11), AvailableMemoryMB=3537 (was 3750) 2024-11-08T06:20:00,338 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=326, ProcessCount=11, AvailableMemoryMB=3537 2024-11-08T06:20:00,340 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51690e43 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47bdcd8b 2024-11-08T06:20:00,343 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x51690e43 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65937a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:00,344 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69a08724 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38d7dcb0 2024-11-08T06:20:00,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d5d6ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:00,350 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:00,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T06:20:00,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:00,352 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:00,352 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-08T06:20:00,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T06:20:00,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741869_1045 (size=420) 2024-11-08T06:20:00,763 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 50376f508b5217860f26c5f8a8fa984c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:00,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741870_1046 (size=71) 2024-11-08T06:20:00,770 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:00,771 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 50376f508b5217860f26c5f8a8fa984c, disabling compactions & flushes 2024-11-08T06:20:00,771 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:20:00,771 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:20:00,771 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. after waiting 0 ms 2024-11-08T06:20:00,771 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:20:00,771 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:20:00,771 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 50376f508b5217860f26c5f8a8fa984c: 2024-11-08T06:20:00,772 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:00,772 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731046800772"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046800772"}]},"ts":"1731046800772"} 2024-11-08T06:20:00,774 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:00,775 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:00,775 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046800775"}]},"ts":"1731046800775"} 2024-11-08T06:20:00,776 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:00,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=50376f508b5217860f26c5f8a8fa984c, ASSIGN}] 2024-11-08T06:20:00,781 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=50376f508b5217860f26c5f8a8fa984c, ASSIGN 2024-11-08T06:20:00,781 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=50376f508b5217860f26c5f8a8fa984c, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:00,932 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=50376f508b5217860f26c5f8a8fa984c, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:00,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 50376f508b5217860f26c5f8a8fa984c, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:01,086 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:01,092 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:20:01,092 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => 50376f508b5217860f26c5f8a8fa984c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:01,093 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:20:01,093 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:01,093 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for 50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:20:01,093 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for 50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:20:01,095 INFO [StoreOpener-50376f508b5217860f26c5f8a8fa984c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:20:01,096 INFO [StoreOpener-50376f508b5217860f26c5f8a8fa984c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50376f508b5217860f26c5f8a8fa984c columnFamilyName testFamily 2024-11-08T06:20:01,096 DEBUG [StoreOpener-50376f508b5217860f26c5f8a8fa984c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:01,097 INFO [StoreOpener-50376f508b5217860f26c5f8a8fa984c-1 {}] regionserver.HStore(327): Store=50376f508b5217860f26c5f8a8fa984c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:01,098 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:20:01,098 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:20:01,101 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for 50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:20:01,103 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:01,104 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened 50376f508b5217860f26c5f8a8fa984c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74964681, jitterRate=0.11706079542636871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:01,105 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for 50376f508b5217860f26c5f8a8fa984c: 2024-11-08T06:20:01,106 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c., pid=59, masterSystemTime=1731046801086 2024-11-08T06:20:01,107 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:20:01,108 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:20:01,108 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=50376f508b5217860f26c5f8a8fa984c, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:01,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T06:20:01,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 50376f508b5217860f26c5f8a8fa984c, server=d1f4bb6aa14f,36803,1731046638998 in 176 msec 2024-11-08T06:20:01,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T06:20:01,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=50376f508b5217860f26c5f8a8fa984c, ASSIGN in 331 msec 2024-11-08T06:20:01,113 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:01,113 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046801113"}]},"ts":"1731046801113"} 2024-11-08T06:20:01,114 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:01,117 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:01,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 767 msec 2024-11-08T06:20:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T06:20:01,355 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-08T06:20:01,355 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:01,355 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:01,359 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:01,360 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:01,360 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T06:20:01,362 DEBUG [hconnection-0x6c808ba6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:01,363 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42986, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:01,370 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:01,372 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:20:01,373 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:20:01,374 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x51690e43 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:20:01,374 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x51690e43 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:01,375 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43002, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:01,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:20:01,377 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:20:01,377 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567406 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:01,378 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:20:01,378 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567406 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:01,378 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T06:20:02,048 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567348 expired clientIPAndPort=172.17.0.2:39178, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:20:02,072 ERROR [LeaseRenewer:jenkins@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:45761,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:20:04,289 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-08T06:20:04,831 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:20:06,639 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:20:08,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-08T06:20:08,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:10,378 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9000 2024-11-08T06:20:10,378 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69a08724 to 127.0.0.1:49997 2024-11-08T06:20:10,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:10,378 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567406 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:20:10,379 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:10,379 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:10,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:10,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567406' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T06:20:10,379 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x51690e43 to 127.0.0.1:49997 2024-11-08T06:20:10,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43002 deadline: 1731046816378, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567406' does not exist 2024-11-08T06:20:10,382 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:20:10,379 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d1f4bb6aa14f,36803,1731046638998 for closing scanner id = 8767934147168567406 for 50376f508b5217860f26c5f8a8fa984c of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T06:20:10,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=230 (was 227) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 326), ProcessCount=11 (was 11), AvailableMemoryMB=3389 (was 3537) 2024-11-08T06:20:10,397 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=3388 2024-11-08T06:20:10,399 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e04c995 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12577bff 2024-11-08T06:20:10,402 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x7e04c995 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6aa8b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:10,403 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x23bd8f1a to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@621bbbb0 2024-11-08T06:20:10,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a8b8e07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:10,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T06:20:10,411 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:10,411 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:10,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-08T06:20:10,412 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T06:20:10,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741871_1047 (size=431) 2024-11-08T06:20:10,824 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f33aa95219c15aed37fc2fd9cd667ba3, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:10,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741872_1048 (size=82) 2024-11-08T06:20:10,879 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:20:11,232 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:11,233 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f33aa95219c15aed37fc2fd9cd667ba3, disabling compactions & flushes 2024-11-08T06:20:11,233 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:20:11,233 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:20:11,233 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. after waiting 0 ms 2024-11-08T06:20:11,233 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:20:11,233 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:20:11,233 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f33aa95219c15aed37fc2fd9cd667ba3: 2024-11-08T06:20:11,234 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:11,234 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731046811234"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046811234"}]},"ts":"1731046811234"} 2024-11-08T06:20:11,236 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:11,237 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:11,237 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046811237"}]},"ts":"1731046811237"} 2024-11-08T06:20:11,238 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:11,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f33aa95219c15aed37fc2fd9cd667ba3, ASSIGN}] 2024-11-08T06:20:11,243 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f33aa95219c15aed37fc2fd9cd667ba3, ASSIGN 2024-11-08T06:20:11,243 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f33aa95219c15aed37fc2fd9cd667ba3, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:11,394 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=f33aa95219c15aed37fc2fd9cd667ba3, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:11,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure f33aa95219c15aed37fc2fd9cd667ba3, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T06:20:11,548 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:11,553 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:20:11,553 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => f33aa95219c15aed37fc2fd9cd667ba3, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:11,554 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:20:11,554 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:11,554 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:20:11,554 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:20:11,556 INFO [StoreOpener-f33aa95219c15aed37fc2fd9cd667ba3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:20:11,558 INFO [StoreOpener-f33aa95219c15aed37fc2fd9cd667ba3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f33aa95219c15aed37fc2fd9cd667ba3 columnFamilyName testFamily 2024-11-08T06:20:11,558 DEBUG [StoreOpener-f33aa95219c15aed37fc2fd9cd667ba3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:11,559 INFO [StoreOpener-f33aa95219c15aed37fc2fd9cd667ba3-1 {}] regionserver.HStore(327): Store=f33aa95219c15aed37fc2fd9cd667ba3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:11,560 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:20:11,560 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:20:11,563 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:20:11,567 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:11,567 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened f33aa95219c15aed37fc2fd9cd667ba3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73670552, jitterRate=0.09777677059173584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:11,568 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for f33aa95219c15aed37fc2fd9cd667ba3: 2024-11-08T06:20:11,569 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3., pid=62, masterSystemTime=1731046811547 2024-11-08T06:20:11,571 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:20:11,571 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:20:11,571 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=f33aa95219c15aed37fc2fd9cd667ba3, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:11,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-08T06:20:11,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure f33aa95219c15aed37fc2fd9cd667ba3, server=d1f4bb6aa14f,36803,1731046638998 in 177 msec 2024-11-08T06:20:11,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T06:20:11,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f33aa95219c15aed37fc2fd9cd667ba3, ASSIGN in 332 msec 2024-11-08T06:20:11,576 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:11,577 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046811576"}]},"ts":"1731046811576"} 2024-11-08T06:20:11,578 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:11,581 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:11,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.1720 sec 2024-11-08T06:20:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-08T06:20:13,416 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-08T06:20:13,416 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:13,416 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:13,420 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:13,420 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:13,420 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-08T06:20:13,422 DEBUG [hconnection-0x5c4adcf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:13,423 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53392, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:13,431 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:13,433 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T06:20:13,434 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T06:20:13,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567410 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:13,436 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T06:20:13,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567410 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:13,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567410 2024-11-08T06:20:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53392 deadline: 1731046816436, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:20:13,437 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:20:13,439 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-08T06:20:13,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567411 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:13,440 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-08T06:20:13,441 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:20:13,442 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:20:13,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567412 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:13,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=8767934147168567412 2024-11-08T06:20:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53392 deadline: 1731046816442, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:20:13,443 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:20:13,444 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:20:13,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567413 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:13,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=8767934147168567413 2024-11-08T06:20:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53392 deadline: 1731046816445, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:20:13,446 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:20:13,447 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:20:13,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567414 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:13,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=8767934147168567414 2024-11-08T06:20:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53392 deadline: 1731046816447, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T06:20:13,448 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:20:13,450 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-08T06:20:13,450 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x23bd8f1a to 127.0.0.1:49997 2024-11-08T06:20:13,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:13,450 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:13,451 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:13,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:13,451 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e04c995 to 127.0.0.1:49997 2024-11-08T06:20:13,460 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=234 (was 230) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 283), ProcessCount=11 (was 11), AvailableMemoryMB=3318 (was 3388) 2024-11-08T06:20:13,468 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=3317 2024-11-08T06:20:13,470 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1233cc50 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b9a91e6 2024-11-08T06:20:13,473 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1233cc50 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18689fc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:13,474 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54a1a7f1 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@811873a 2024-11-08T06:20:13,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c9b24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:13,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T06:20:13,481 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:13,481 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:13,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-08T06:20:13,482 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T06:20:13,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741873_1049 (size=425) 2024-11-08T06:20:13,891 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ea36b5332d3c79bdc64235d12d2c6bf0, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:13,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741874_1050 (size=76) 2024-11-08T06:20:14,299 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:14,299 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing ea36b5332d3c79bdc64235d12d2c6bf0, disabling compactions & flushes 2024-11-08T06:20:14,299 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:20:14,299 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:20:14,299 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. after waiting 0 ms 2024-11-08T06:20:14,299 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:20:14,299 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:20:14,299 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for ea36b5332d3c79bdc64235d12d2c6bf0: 2024-11-08T06:20:14,300 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:14,301 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731046814300"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046814300"}]},"ts":"1731046814300"} 2024-11-08T06:20:14,302 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:14,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:14,303 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046814303"}]},"ts":"1731046814303"} 2024-11-08T06:20:14,305 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:14,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ea36b5332d3c79bdc64235d12d2c6bf0, ASSIGN}] 2024-11-08T06:20:14,309 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ea36b5332d3c79bdc64235d12d2c6bf0, ASSIGN 2024-11-08T06:20:14,310 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ea36b5332d3c79bdc64235d12d2c6bf0, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:14,358 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567361 expired clientIPAndPort=172.17.0.2:40546, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:20:14,460 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=ea36b5332d3c79bdc64235d12d2c6bf0, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:14,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure ea36b5332d3c79bdc64235d12d2c6bf0, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T06:20:14,615 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:14,619 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:20:14,619 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => ea36b5332d3c79bdc64235d12d2c6bf0, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:14,620 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:20:14,620 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:14,620 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:20:14,620 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:20:14,622 INFO [StoreOpener-ea36b5332d3c79bdc64235d12d2c6bf0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:20:14,623 INFO [StoreOpener-ea36b5332d3c79bdc64235d12d2c6bf0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea36b5332d3c79bdc64235d12d2c6bf0 columnFamilyName testFamily 2024-11-08T06:20:14,623 DEBUG [StoreOpener-ea36b5332d3c79bdc64235d12d2c6bf0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:14,624 INFO [StoreOpener-ea36b5332d3c79bdc64235d12d2c6bf0-1 {}] regionserver.HStore(327): Store=ea36b5332d3c79bdc64235d12d2c6bf0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:14,625 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:20:14,625 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:20:14,627 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:20:14,630 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:14,630 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened ea36b5332d3c79bdc64235d12d2c6bf0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75294675, jitterRate=0.12197808921337128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:14,632 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for ea36b5332d3c79bdc64235d12d2c6bf0: 2024-11-08T06:20:14,632 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0., pid=65, masterSystemTime=1731046814614 2024-11-08T06:20:14,634 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:20:14,634 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:20:14,635 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=ea36b5332d3c79bdc64235d12d2c6bf0, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:14,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T06:20:14,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure ea36b5332d3c79bdc64235d12d2c6bf0, server=d1f4bb6aa14f,36803,1731046638998 in 173 msec 2024-11-08T06:20:14,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-08T06:20:14,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=ea36b5332d3c79bdc64235d12d2c6bf0, ASSIGN in 330 msec 2024-11-08T06:20:14,640 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:14,640 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046814640"}]},"ts":"1731046814640"} 2024-11-08T06:20:14,642 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:14,644 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:14,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.1650 sec 2024-11-08T06:20:16,291 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-08T06:20:16,292 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-08T06:20:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T06:20:16,485 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-08T06:20:16,485 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:16,486 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:16,489 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:16,489 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:16,489 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T06:20:16,491 DEBUG [hconnection-0x6f8d83f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:16,493 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53408, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:16,500 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:16,502 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:20:16,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:20:16,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-08T06:20:17,055 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:20:17,403 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:20:18,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T06:20:18,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:20:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53408 deadline: 1731046819504, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:20:18,519 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:20:18,519 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567419 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:20:18,520 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2018 2024-11-08T06:20:18,521 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x54a1a7f1 to 127.0.0.1:49997 2024-11-08T06:20:18,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:18,521 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:18,521 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:18,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:18,521 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1233cc50 to 127.0.0.1:49997 2024-11-08T06:20:18,530 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=233 (was 234), OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=240 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=3290 (was 3317) 2024-11-08T06:20:18,539 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=240, ProcessCount=11, AvailableMemoryMB=3290 2024-11-08T06:20:18,540 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5377cf12 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29501aba 2024-11-08T06:20:18,543 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x5377cf12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295d7787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:18,544 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53cd61a6 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7377eaef 2024-11-08T06:20:18,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e15b4cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:18,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T06:20:18,551 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:18,551 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:18,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-08T06:20:18,552 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T06:20:18,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741875_1051 (size=426) 2024-11-08T06:20:18,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T06:20:18,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:18,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-08T06:20:18,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:18,963 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8443cee1b4a7317c15452e64a36df3da, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:18,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741876_1052 (size=77) 2024-11-08T06:20:19,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:19,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 8443cee1b4a7317c15452e64a36df3da, disabling compactions & flushes 2024-11-08T06:20:19,371 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:20:19,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:20:19,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. after waiting 0 ms 2024-11-08T06:20:19,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:20:19,371 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:20:19,371 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8443cee1b4a7317c15452e64a36df3da: 2024-11-08T06:20:19,372 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:19,373 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731046819372"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046819372"}]},"ts":"1731046819372"} 2024-11-08T06:20:19,374 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:19,375 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:19,375 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046819375"}]},"ts":"1731046819375"} 2024-11-08T06:20:19,376 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:19,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8443cee1b4a7317c15452e64a36df3da, ASSIGN}] 2024-11-08T06:20:19,381 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8443cee1b4a7317c15452e64a36df3da, ASSIGN 2024-11-08T06:20:19,381 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8443cee1b4a7317c15452e64a36df3da, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:19,532 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=8443cee1b4a7317c15452e64a36df3da, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:19,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 8443cee1b4a7317c15452e64a36df3da, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T06:20:19,685 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:19,690 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:20:19,690 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 8443cee1b4a7317c15452e64a36df3da, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:19,691 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:20:19,691 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:19,691 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:20:19,691 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:20:19,693 INFO [StoreOpener-8443cee1b4a7317c15452e64a36df3da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:20:19,695 INFO [StoreOpener-8443cee1b4a7317c15452e64a36df3da-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8443cee1b4a7317c15452e64a36df3da columnFamilyName testFamily 2024-11-08T06:20:19,695 DEBUG [StoreOpener-8443cee1b4a7317c15452e64a36df3da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:19,696 INFO [StoreOpener-8443cee1b4a7317c15452e64a36df3da-1 {}] regionserver.HStore(327): Store=8443cee1b4a7317c15452e64a36df3da/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:19,697 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:20:19,697 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:20:19,700 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:20:19,703 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:19,704 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 8443cee1b4a7317c15452e64a36df3da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62975657, jitterRate=-0.06158958375453949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:19,705 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 8443cee1b4a7317c15452e64a36df3da: 2024-11-08T06:20:19,706 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da., pid=68, masterSystemTime=1731046819685 2024-11-08T06:20:19,708 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:20:19,708 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:20:19,708 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=8443cee1b4a7317c15452e64a36df3da, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:19,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-08T06:20:19,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 8443cee1b4a7317c15452e64a36df3da, server=d1f4bb6aa14f,36803,1731046638998 in 176 msec 2024-11-08T06:20:19,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T06:20:19,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=8443cee1b4a7317c15452e64a36df3da, ASSIGN in 331 msec 2024-11-08T06:20:19,713 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:19,713 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046819713"}]},"ts":"1731046819713"} 2024-11-08T06:20:19,714 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:19,717 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:19,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1680 sec 2024-11-08T06:20:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-08T06:20:21,556 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-08T06:20:21,556 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:21,556 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:21,560 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:21,560 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:21,560 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-08T06:20:21,563 DEBUG [hconnection-0x702da386-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:21,564 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57724, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:21,571 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:21,574 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:20:21,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:20:21,964 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567365 expired clientIPAndPort=172.17.0.2:40556, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:20:22,293 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-08T06:20:23,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 88 bytes 2024-11-08T06:20:23,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:20:24,579 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5023: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5023: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:20:24,579 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-08T06:20:24.579Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5023: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5023: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:20:24,580 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3005 2024-11-08T06:20:24,580 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x53cd61a6 to 127.0.0.1:49997 2024-11-08T06:20:24,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:24,580 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:24,580 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:24,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:24,580 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5377cf12 to 127.0.0.1:49997 2024-11-08T06:20:24,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=230 (was 233), OpenFileDescriptor=451 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 240), ProcessCount=11 (was 11), AvailableMemoryMB=3383 (was 3290) - AvailableMemoryMB LEAK? - 2024-11-08T06:20:24,597 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=230, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=3382 2024-11-08T06:20:24,599 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1383693d to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@199cbc30 2024-11-08T06:20:24,602 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1383693d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc40604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:24,603 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x28227f00 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c952e0b 2024-11-08T06:20:24,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34577225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:24,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T06:20:24,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:24,609 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:24,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-08T06:20:24,610 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T06:20:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741877_1053 (size=424) 2024-11-08T06:20:25,019 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0b865dba842e12e627d69ce50c784684, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:25,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741878_1054 (size=75) 2024-11-08T06:20:25,091 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:20:25,192 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:20:25,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:25,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 0b865dba842e12e627d69ce50c784684, disabling compactions & flushes 2024-11-08T06:20:25,428 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:20:25,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:20:25,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. after waiting 0 ms 2024-11-08T06:20:25,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:20:25,428 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:20:25,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0b865dba842e12e627d69ce50c784684: 2024-11-08T06:20:25,429 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:25,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731046825429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046825429"}]},"ts":"1731046825429"} 2024-11-08T06:20:25,431 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:25,432 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:25,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046825432"}]},"ts":"1731046825432"} 2024-11-08T06:20:25,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:25,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=0b865dba842e12e627d69ce50c784684, ASSIGN}] 2024-11-08T06:20:25,439 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=0b865dba842e12e627d69ce50c784684, ASSIGN 2024-11-08T06:20:25,439 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=0b865dba842e12e627d69ce50c784684, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:25,590 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=0b865dba842e12e627d69ce50c784684, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:25,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure 0b865dba842e12e627d69ce50c784684, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T06:20:25,744 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:25,748 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:20:25,748 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => 0b865dba842e12e627d69ce50c784684, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:25,749 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true 0b865dba842e12e627d69ce50c784684 2024-11-08T06:20:25,749 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:25,749 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for 0b865dba842e12e627d69ce50c784684 2024-11-08T06:20:25,749 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for 0b865dba842e12e627d69ce50c784684 2024-11-08T06:20:25,751 INFO [StoreOpener-0b865dba842e12e627d69ce50c784684-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0b865dba842e12e627d69ce50c784684 2024-11-08T06:20:25,752 INFO [StoreOpener-0b865dba842e12e627d69ce50c784684-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b865dba842e12e627d69ce50c784684 columnFamilyName testFamily 2024-11-08T06:20:25,752 DEBUG [StoreOpener-0b865dba842e12e627d69ce50c784684-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:25,753 INFO [StoreOpener-0b865dba842e12e627d69ce50c784684-1 {}] regionserver.HStore(327): Store=0b865dba842e12e627d69ce50c784684/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:25,754 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684 2024-11-08T06:20:25,754 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684 2024-11-08T06:20:25,756 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for 0b865dba842e12e627d69ce50c784684 2024-11-08T06:20:25,759 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:25,759 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened 0b865dba842e12e627d69ce50c784684; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71206699, jitterRate=0.06106249988079071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:25,760 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for 0b865dba842e12e627d69ce50c784684: 2024-11-08T06:20:25,761 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684., pid=71, masterSystemTime=1731046825744 2024-11-08T06:20:25,763 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:20:25,763 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:20:25,763 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=0b865dba842e12e627d69ce50c784684, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:25,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T06:20:25,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure 0b865dba842e12e627d69ce50c784684, server=d1f4bb6aa14f,36803,1731046638998 in 173 msec 2024-11-08T06:20:25,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-08T06:20:25,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=0b865dba842e12e627d69ce50c784684, ASSIGN in 328 msec 2024-11-08T06:20:25,768 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:25,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046825768"}]},"ts":"1731046825768"} 2024-11-08T06:20:25,769 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:25,771 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:25,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.1630 sec 2024-11-08T06:20:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-08T06:20:27,613 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-08T06:20:27,613 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:27,614 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:27,617 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:27,618 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:27,618 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-08T06:20:27,620 DEBUG [hconnection-0x698e946a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:27,621 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57736, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:27,628 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:27,630 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:20:27,630 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:20:27,631 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1383693d {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:20:27,631 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1383693d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:27,632 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57750, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:20:27,634 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:20:27,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567431 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:27,635 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:20:27,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567431 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:27,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T06:20:28,295 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-08T06:20:28,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-08T06:20:28,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:28,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-08T06:20:28,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:29,571 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567371 expired clientIPAndPort=172.17.0.2:53426, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:20:30,639 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3004 2024-11-08T06:20:30,639 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x28227f00 to 127.0.0.1:49997 2024-11-08T06:20:30,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:30,640 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:30,639 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567431 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:20:30,640 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:30,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:30,640 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1383693d to 127.0.0.1:49997 2024-11-08T06:20:30,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567431' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T06:20:30,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57750 deadline: 1731046831639, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8767934147168567431' does not exist 2024-11-08T06:20:30,640 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d1f4bb6aa14f,36803,1731046638998 for closing scanner id = 8767934147168567431 for 0b865dba842e12e627d69ce50c784684 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-08T06:20:30,643 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:20:30,650 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=230 (was 230), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=203 (was 220), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=3349 (was 3382) 2024-11-08T06:20:30,659 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=230, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=203, ProcessCount=12, AvailableMemoryMB=3348 2024-11-08T06:20:30,661 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x67a65bdd to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74c9aadd 2024-11-08T06:20:30,664 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x67a65bdd {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705b420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:30,665 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7618553c to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24985dce 2024-11-08T06:20:30,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71358233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:30,671 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:30,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-08T06:20:30,674 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:30,674 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:30,674 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-08T06:20:30,675 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T06:20:30,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741879_1055 (size=416) 2024-11-08T06:20:31,085 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 52eec85d1f18d9e6fb9f54d6a6190ed7, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:31,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741880_1056 (size=67) 2024-11-08T06:20:31,105 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:45761,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:20:31,136 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:20:31,250 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:20:31,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:31,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 52eec85d1f18d9e6fb9f54d6a6190ed7, disabling compactions & flushes 2024-11-08T06:20:31,493 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:20:31,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:20:31,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. after waiting 0 ms 2024-11-08T06:20:31,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:20:31,493 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:20:31,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 52eec85d1f18d9e6fb9f54d6a6190ed7: 2024-11-08T06:20:31,494 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:31,494 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731046831494"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046831494"}]},"ts":"1731046831494"} 2024-11-08T06:20:31,496 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:31,496 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:31,497 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046831496"}]},"ts":"1731046831496"} 2024-11-08T06:20:31,498 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:31,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=52eec85d1f18d9e6fb9f54d6a6190ed7, ASSIGN}] 2024-11-08T06:20:31,502 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=52eec85d1f18d9e6fb9f54d6a6190ed7, ASSIGN 2024-11-08T06:20:31,503 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=52eec85d1f18d9e6fb9f54d6a6190ed7, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:31,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=52eec85d1f18d9e6fb9f54d6a6190ed7, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:31,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 52eec85d1f18d9e6fb9f54d6a6190ed7, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T06:20:31,807 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:31,812 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:20:31,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 52eec85d1f18d9e6fb9f54d6a6190ed7, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:31,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:20:31,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:31,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:20:31,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:20:31,814 INFO [StoreOpener-52eec85d1f18d9e6fb9f54d6a6190ed7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:20:31,816 INFO [StoreOpener-52eec85d1f18d9e6fb9f54d6a6190ed7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52eec85d1f18d9e6fb9f54d6a6190ed7 columnFamilyName testFamily 2024-11-08T06:20:31,816 DEBUG [StoreOpener-52eec85d1f18d9e6fb9f54d6a6190ed7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:31,817 INFO [StoreOpener-52eec85d1f18d9e6fb9f54d6a6190ed7-1 {}] regionserver.HStore(327): Store=52eec85d1f18d9e6fb9f54d6a6190ed7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:31,817 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:20:31,818 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:20:31,820 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:20:31,822 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:31,823 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 52eec85d1f18d9e6fb9f54d6a6190ed7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75111317, jitterRate=0.11924584209918976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:31,824 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 52eec85d1f18d9e6fb9f54d6a6190ed7: 2024-11-08T06:20:31,824 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7., pid=74, masterSystemTime=1731046831807 2024-11-08T06:20:31,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:20:31,826 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:20:31,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=52eec85d1f18d9e6fb9f54d6a6190ed7, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:31,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-08T06:20:31,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 52eec85d1f18d9e6fb9f54d6a6190ed7, server=d1f4bb6aa14f,36803,1731046638998 in 173 msec 2024-11-08T06:20:31,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T06:20:31,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=52eec85d1f18d9e6fb9f54d6a6190ed7, ASSIGN in 328 msec 2024-11-08T06:20:31,831 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:31,831 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046831831"}]},"ts":"1731046831831"} 2024-11-08T06:20:31,832 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:31,835 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:31,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.1640 sec 2024-11-08T06:20:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-08T06:20:33,678 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-08T06:20:33,678 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:33,679 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:33,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-08T06:20:33,685 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567432 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:33,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:33,687 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:33,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-08T06:20:33,689 DEBUG [hconnection-0x15b2237d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:33,690 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43736, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:33,697 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:33,699 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:20:33,700 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567435 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:20:33,701 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-08T06:20:34,296 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-08T06:20:36,577 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567373 expired clientIPAndPort=172.17.0.2:53426, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:20:37,314 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:20:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T06:20:38,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:40,201 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-11-08T06:20:40,201 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7618553c to 127.0.0.1:49997 2024-11-08T06:20:40,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:40,202 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:40,202 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:40,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:40,202 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x67a65bdd to 127.0.0.1:49997 2024-11-08T06:20:40,211 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=230 (was 230), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=171 (was 203), ProcessCount=11 (was 12), AvailableMemoryMB=3303 (was 3348) 2024-11-08T06:20:40,219 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=171, ProcessCount=11, AvailableMemoryMB=3303 2024-11-08T06:20:40,221 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1cc98284 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@fc5634d 2024-11-08T06:20:40,224 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x1cc98284 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648be025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:40,225 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1bc81d54 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@bd52c38 2024-11-08T06:20:40,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd85264, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:40,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:40,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-08T06:20:40,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:40,231 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:40,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-08T06:20:40,232 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T06:20:40,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741881_1057 (size=420) 2024-11-08T06:20:40,417 ERROR [LeaseRenewer:jenkins@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:45761,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:20:40,641 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3dfc4ceec119f4060839de57ca17c059, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:40,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741882_1058 (size=71) 2024-11-08T06:20:41,048 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:41,049 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 3dfc4ceec119f4060839de57ca17c059, disabling compactions & flushes 2024-11-08T06:20:41,049 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:20:41,049 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:20:41,049 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. after waiting 0 ms 2024-11-08T06:20:41,049 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:20:41,049 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:20:41,049 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3dfc4ceec119f4060839de57ca17c059: 2024-11-08T06:20:41,050 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:41,050 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731046841050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046841050"}]},"ts":"1731046841050"} 2024-11-08T06:20:41,052 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:41,052 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:41,053 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046841052"}]},"ts":"1731046841052"} 2024-11-08T06:20:41,054 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:41,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=3dfc4ceec119f4060839de57ca17c059, ASSIGN}] 2024-11-08T06:20:41,061 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=3dfc4ceec119f4060839de57ca17c059, ASSIGN 2024-11-08T06:20:41,061 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=3dfc4ceec119f4060839de57ca17c059, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:41,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=3dfc4ceec119f4060839de57ca17c059, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:41,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 3dfc4ceec119f4060839de57ca17c059, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T06:20:41,365 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:41,370 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:20:41,370 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => 3dfc4ceec119f4060839de57ca17c059, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:41,370 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:20:41,371 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:41,371 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for 3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:20:41,371 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for 3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:20:41,372 INFO [StoreOpener-3dfc4ceec119f4060839de57ca17c059-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:20:41,374 INFO [StoreOpener-3dfc4ceec119f4060839de57ca17c059-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3dfc4ceec119f4060839de57ca17c059 columnFamilyName testFamily 2024-11-08T06:20:41,374 DEBUG [StoreOpener-3dfc4ceec119f4060839de57ca17c059-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:41,374 INFO [StoreOpener-3dfc4ceec119f4060839de57ca17c059-1 {}] regionserver.HStore(327): Store=3dfc4ceec119f4060839de57ca17c059/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:41,375 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:20:41,376 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:20:41,378 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for 3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:20:41,380 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:41,380 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened 3dfc4ceec119f4060839de57ca17c059; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61032881, jitterRate=-0.09053920209407806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:41,381 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for 3dfc4ceec119f4060839de57ca17c059: 2024-11-08T06:20:41,382 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059., pid=77, masterSystemTime=1731046841365 2024-11-08T06:20:41,384 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:20:41,384 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:20:41,384 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=3dfc4ceec119f4060839de57ca17c059, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:41,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T06:20:41,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 3dfc4ceec119f4060839de57ca17c059, server=d1f4bb6aa14f,36803,1731046638998 in 171 msec 2024-11-08T06:20:41,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-08T06:20:41,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=3dfc4ceec119f4060839de57ca17c059, ASSIGN in 328 msec 2024-11-08T06:20:41,389 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:41,389 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046841389"}]},"ts":"1731046841389"} 2024-11-08T06:20:41,391 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:41,393 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:41,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1630 sec 2024-11-08T06:20:41,681 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567378 expired clientIPAndPort=172.17.0.2:33110, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:20:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T06:20:43,235 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-08T06:20:43,235 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:43,236 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:43,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-08T06:20:43,240 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567436 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:43,241 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:43,241 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:43,241 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-08T06:20:43,243 DEBUG [hconnection-0x7c34f73b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:43,244 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45184, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:43,251 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:43,253 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:20:43,254 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567439 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-08T06:20:43,254 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-08T06:20:44,755 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-08T06:20:44,755 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1bc81d54 to 127.0.0.1:49997 2024-11-08T06:20:44,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:44,755 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:44,755 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:44,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:44,756 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1cc98284 to 127.0.0.1:49997 2024-11-08T06:20:44,765 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 230) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 171), ProcessCount=9 (was 11), AvailableMemoryMB=4334 (was 3303) - AvailableMemoryMB LEAK? - 2024-11-08T06:20:44,773 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=9, AvailableMemoryMB=4333 2024-11-08T06:20:44,774 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x013266da to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42e2d266 2024-11-08T06:20:44,778 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x013266da {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5640835a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:44,778 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21e8478e to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69665769 2024-11-08T06:20:44,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b70e829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:44,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-08T06:20:44,785 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:44,785 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:44,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-08T06:20:44,786 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T06:20:44,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741883_1059 (size=419) 2024-11-08T06:20:45,196 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => faef91a41733387bbcd9cb53b34883eb, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:45,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741884_1060 (size=70) 2024-11-08T06:20:45,604 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:45,604 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing faef91a41733387bbcd9cb53b34883eb, disabling compactions & flushes 2024-11-08T06:20:45,605 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:20:45,605 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:20:45,605 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. after waiting 0 ms 2024-11-08T06:20:45,605 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:20:45,605 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:20:45,605 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for faef91a41733387bbcd9cb53b34883eb: 2024-11-08T06:20:45,606 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:45,606 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731046845606"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046845606"}]},"ts":"1731046845606"} 2024-11-08T06:20:45,608 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:45,608 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:45,609 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046845608"}]},"ts":"1731046845608"} 2024-11-08T06:20:45,610 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:45,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=faef91a41733387bbcd9cb53b34883eb, ASSIGN}] 2024-11-08T06:20:45,614 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=faef91a41733387bbcd9cb53b34883eb, ASSIGN 2024-11-08T06:20:45,615 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=faef91a41733387bbcd9cb53b34883eb, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:45,684 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567383 expired clientIPAndPort=172.17.0.2:51498, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:20:45,765 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=faef91a41733387bbcd9cb53b34883eb, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:45,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure faef91a41733387bbcd9cb53b34883eb, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T06:20:45,919 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:45,924 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:20:45,924 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => faef91a41733387bbcd9cb53b34883eb, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:45,924 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:20:45,924 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:45,924 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:20:45,924 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:20:45,926 INFO [StoreOpener-faef91a41733387bbcd9cb53b34883eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:20:45,928 INFO [StoreOpener-faef91a41733387bbcd9cb53b34883eb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region faef91a41733387bbcd9cb53b34883eb columnFamilyName testFamily 2024-11-08T06:20:45,928 DEBUG [StoreOpener-faef91a41733387bbcd9cb53b34883eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:45,928 INFO [StoreOpener-faef91a41733387bbcd9cb53b34883eb-1 {}] regionserver.HStore(327): Store=faef91a41733387bbcd9cb53b34883eb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:45,929 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:20:45,929 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:20:45,931 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:20:45,934 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:45,934 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened faef91a41733387bbcd9cb53b34883eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73979164, jitterRate=0.10237544775009155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:45,935 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for faef91a41733387bbcd9cb53b34883eb: 2024-11-08T06:20:45,936 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb., pid=80, masterSystemTime=1731046845919 2024-11-08T06:20:45,937 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:20:45,937 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:20:45,938 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=faef91a41733387bbcd9cb53b34883eb, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:45,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-08T06:20:45,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure faef91a41733387bbcd9cb53b34883eb, server=d1f4bb6aa14f,36803,1731046638998 in 172 msec 2024-11-08T06:20:45,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T06:20:45,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=faef91a41733387bbcd9cb53b34883eb, ASSIGN in 327 msec 2024-11-08T06:20:45,942 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:45,942 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046845942"}]},"ts":"1731046845942"} 2024-11-08T06:20:45,943 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:45,946 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:45,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1630 sec 2024-11-08T06:20:46,298 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-08T06:20:46,299 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-08T06:20:46,872 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:20:47,403 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:20:47,686 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567385 expired clientIPAndPort=172.17.0.2:51498, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:20:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-08T06:20:47,789 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-08T06:20:47,789 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:47,789 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:47,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-08T06:20:47,793 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567440 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:47,794 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:47,795 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:47,795 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-08T06:20:47,797 DEBUG [hconnection-0x6fded8d5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:47,798 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45190, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:47,804 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:47,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:20:47,807 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:20:47,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567443 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:47,808 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:20:47,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567443 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:47,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-08T06:20:48,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T06:20:48,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:48,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T06:20:48,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:20:50,819 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4017: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4017: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:20:50,819 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3010 2024-11-08T06:20:50,819 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21e8478e to 127.0.0.1:49997 2024-11-08T06:20:50,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:50,819 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:20:50,820 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:20:50,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:20:50,820 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x013266da to 127.0.0.1:49997 2024-11-08T06:20:50,828 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=230 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=152 (was 166), ProcessCount=9 (was 9), AvailableMemoryMB=4307 (was 4333) 2024-11-08T06:20:50,836 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=152, ProcessCount=9, AvailableMemoryMB=4307 2024-11-08T06:20:50,838 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63c0dc13 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@643be889 2024-11-08T06:20:50,841 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x63c0dc13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@719bcc75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:50,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6544344c to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1020cd6d 2024-11-08T06:20:50,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e5ab3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:20:50,846 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:20:50,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T06:20:50,848 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:20:50,848 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:50,848 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-08T06:20:50,849 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:20:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T06:20:50,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741885_1061 (size=421) 2024-11-08T06:20:51,258 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a45c97b247e9558943305207041e58d5, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:20:51,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741886_1062 (size=72) 2024-11-08T06:20:51,309 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:20:51,665 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:51,666 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing a45c97b247e9558943305207041e58d5, disabling compactions & flushes 2024-11-08T06:20:51,666 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:20:51,666 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:20:51,666 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. after waiting 0 ms 2024-11-08T06:20:51,666 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:20:51,666 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:20:51,666 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for a45c97b247e9558943305207041e58d5: 2024-11-08T06:20:51,667 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:20:51,667 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731046851667"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046851667"}]},"ts":"1731046851667"} 2024-11-08T06:20:51,669 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:20:51,669 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:20:51,670 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046851669"}]},"ts":"1731046851669"} 2024-11-08T06:20:51,671 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:20:51,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a45c97b247e9558943305207041e58d5, ASSIGN}] 2024-11-08T06:20:51,675 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a45c97b247e9558943305207041e58d5, ASSIGN 2024-11-08T06:20:51,675 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a45c97b247e9558943305207041e58d5, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:20:51,826 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=a45c97b247e9558943305207041e58d5, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:51,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure a45c97b247e9558943305207041e58d5, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:20:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T06:20:51,979 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:51,983 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:20:51,983 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => a45c97b247e9558943305207041e58d5, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:20:51,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true a45c97b247e9558943305207041e58d5 2024-11-08T06:20:51,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:20:51,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for a45c97b247e9558943305207041e58d5 2024-11-08T06:20:51,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for a45c97b247e9558943305207041e58d5 2024-11-08T06:20:51,985 INFO [StoreOpener-a45c97b247e9558943305207041e58d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a45c97b247e9558943305207041e58d5 2024-11-08T06:20:51,987 INFO [StoreOpener-a45c97b247e9558943305207041e58d5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a45c97b247e9558943305207041e58d5 columnFamilyName testFamily 2024-11-08T06:20:51,987 DEBUG [StoreOpener-a45c97b247e9558943305207041e58d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:20:51,987 INFO [StoreOpener-a45c97b247e9558943305207041e58d5-1 {}] regionserver.HStore(327): Store=a45c97b247e9558943305207041e58d5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:20:51,988 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5 2024-11-08T06:20:51,989 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5 2024-11-08T06:20:51,991 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for a45c97b247e9558943305207041e58d5 2024-11-08T06:20:51,993 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:20:51,994 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened a45c97b247e9558943305207041e58d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73132400, jitterRate=0.08975768089294434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:20:51,995 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for a45c97b247e9558943305207041e58d5: 2024-11-08T06:20:51,995 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5., pid=83, masterSystemTime=1731046851979 2024-11-08T06:20:51,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:20:51,997 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:20:51,997 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=a45c97b247e9558943305207041e58d5, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:20:52,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T06:20:52,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure a45c97b247e9558943305207041e58d5, server=d1f4bb6aa14f,36803,1731046638998 in 172 msec 2024-11-08T06:20:52,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-08T06:20:52,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=a45c97b247e9558943305207041e58d5, ASSIGN in 326 msec 2024-11-08T06:20:52,002 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:20:52,002 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046852002"}]},"ts":"1731046852002"} 2024-11-08T06:20:52,003 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:20:52,005 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:20:52,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1590 sec 2024-11-08T06:20:52,300 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-08T06:20:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T06:20:53,852 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-08T06:20:53,852 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:20:53,852 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:53,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-08T06:20:53,857 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567444 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:20:53,858 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-08T06:20:53,858 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:20:53,858 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-08T06:20:53,860 DEBUG [hconnection-0x10b041f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:20:53,862 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54236, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:20:53,869 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:20:53,871 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:20:53,872 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:20:53,873 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-08T06:20:57,527 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:20:57,893 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567399 expired clientIPAndPort=172.17.0.2:41540, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:20:58,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T06:20:58,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:21:00,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 11 bytes 2024-11-08T06:21:00,892 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:21:00,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54236 deadline: 1731046862872, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-08T06:21:00,892 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-08T06:21:00,893 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 8767934147168567447 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T06:21:00,893 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7022 2024-11-08T06:21:00,894 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6544344c to 127.0.0.1:49997 2024-11-08T06:21:00,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:00,894 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:21:00,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:21:00,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:00,894 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63c0dc13 to 127.0.0.1:49997 2024-11-08T06:21:00,904 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230 (was 230), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=129 (was 152), ProcessCount=9 (was 9), AvailableMemoryMB=4216 (was 4307) 2024-11-08T06:21:00,912 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=9, AvailableMemoryMB=4215 2024-11-08T06:21:00,913 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x796d70bb to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@773f54fa 2024-11-08T06:21:00,916 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x796d70bb {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@777f2809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:21:00,917 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x722eaa14 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4dc7636a 2024-11-08T06:21:00,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3d3ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:21:00,922 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:21:00,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T06:21:00,923 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:21:00,923 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:21:00,923 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-08T06:21:00,924 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:21:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T06:21:00,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741887_1063 (size=427) 2024-11-08T06:21:01,336 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a99ec1db5991df8c4fc52125fca5b6cb, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:21:01,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741888_1064 (size=78) 2024-11-08T06:21:01,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:21:01,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing a99ec1db5991df8c4fc52125fca5b6cb, disabling compactions & flushes 2024-11-08T06:21:01,743 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:01,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:01,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. after waiting 0 ms 2024-11-08T06:21:01,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:01,743 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:01,743 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for a99ec1db5991df8c4fc52125fca5b6cb: 2024-11-08T06:21:01,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:21:01,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731046861744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046861744"}]},"ts":"1731046861744"} 2024-11-08T06:21:01,746 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:21:01,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:21:01,747 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046861747"}]},"ts":"1731046861747"} 2024-11-08T06:21:01,748 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:21:01,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a99ec1db5991df8c4fc52125fca5b6cb, ASSIGN}] 2024-11-08T06:21:01,752 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a99ec1db5991df8c4fc52125fca5b6cb, ASSIGN 2024-11-08T06:21:01,753 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a99ec1db5991df8c4fc52125fca5b6cb, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:21:01,824 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:45761,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:21:01,903 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=a99ec1db5991df8c4fc52125fca5b6cb, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:01,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure a99ec1db5991df8c4fc52125fca5b6cb, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:21:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T06:21:02,056 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:02,062 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:02,062 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => a99ec1db5991df8c4fc52125fca5b6cb, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:21:02,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:02,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:21:02,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:02,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:02,065 INFO [StoreOpener-a99ec1db5991df8c4fc52125fca5b6cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:02,066 INFO [StoreOpener-a99ec1db5991df8c4fc52125fca5b6cb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a99ec1db5991df8c4fc52125fca5b6cb columnFamilyName testFamily 2024-11-08T06:21:02,066 DEBUG [StoreOpener-a99ec1db5991df8c4fc52125fca5b6cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:21:02,067 INFO [StoreOpener-a99ec1db5991df8c4fc52125fca5b6cb-1 {}] regionserver.HStore(327): Store=a99ec1db5991df8c4fc52125fca5b6cb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:21:02,067 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:02,068 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:02,069 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:02,072 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:21:02,072 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened a99ec1db5991df8c4fc52125fca5b6cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59981268, jitterRate=-0.10620945692062378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:21:02,073 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for a99ec1db5991df8c4fc52125fca5b6cb: 2024-11-08T06:21:02,073 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., pid=86, masterSystemTime=1731046862056 2024-11-08T06:21:02,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:02,075 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:02,075 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=a99ec1db5991df8c4fc52125fca5b6cb, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:02,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T06:21:02,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure a99ec1db5991df8c4fc52125fca5b6cb, server=d1f4bb6aa14f,36803,1731046638998 in 171 msec 2024-11-08T06:21:02,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-08T06:21:02,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a99ec1db5991df8c4fc52125fca5b6cb, ASSIGN in 326 msec 2024-11-08T06:21:02,079 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:21:02,079 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046862079"}]},"ts":"1731046862079"} 2024-11-08T06:21:02,080 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:21:02,083 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:21:02,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1610 sec 2024-11-08T06:21:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-08T06:21:03,927 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-08T06:21:03,927 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:21:03,927 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:21:03,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-08T06:21:03,931 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567449 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:21:03,932 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T06:21:03,932 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:21:03,932 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T06:21:03,934 DEBUG [hconnection-0x677467d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:21:03,935 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44998, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:21:03,942 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:21:03,943 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:21:03,944 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:21:03,945 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x796d70bb {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:21:03,945 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x796d70bb {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:21:03,946 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45014, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:21:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:21:03,947 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:21:04,302 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T06:21:04,901 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567401 expired clientIPAndPort=172.17.0.2:41540, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:21:07,564 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:21:08,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T06:21:08,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:21:09,948 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms 2024-11-08T06:21:09,949 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms 2024-11-08T06:21:09,949 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:21:10,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T06:21:10,646 ERROR [LeaseRenewer:jenkins@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:45761,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:21:10,905 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567406 expired clientIPAndPort=172.17.0.2:43002, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:21:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:21:10,965 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-08T06:21:12,948 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1979ms 2024-11-08T06:21:12,948 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1979ms 2024-11-08T06:21:12,949 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:21:12,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T06:21:09.949Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms 2024-11-08T06:21:12.949Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1979ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1983ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T06:21:12,950 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-08T06:21:12,950 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x722eaa14 to 127.0.0.1:49997 2024-11-08T06:21:12,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:12,950 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:21:12,950 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:21:12,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:12,951 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x796d70bb to 127.0.0.1:49997 2024-11-08T06:21:12,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=226 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=100 (was 129), ProcessCount=9 (was 9), AvailableMemoryMB=4212 (was 4215) 2024-11-08T06:21:12,967 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=100, ProcessCount=9, AvailableMemoryMB=4212 2024-11-08T06:21:12,968 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7268f750 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d2a6ef4 2024-11-08T06:21:12,973 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x7268f750 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d751285, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:21:12,974 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2111251f to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1455ce52 2024-11-08T06:21:12,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213279ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:21:12,979 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:21:12,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-08T06:21:12,981 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:21:12,981 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:21:12,981 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-08T06:21:12,982 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:21:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T06:21:12,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741889_1065 (size=415) 2024-11-08T06:21:13,392 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f313d7cca28bccec72c3e8e677b94d37, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:21:13,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741890_1066 (size=66) 2024-11-08T06:21:13,507 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567410 expired clientIPAndPort=172.17.0.2:53392, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:13,507 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567413 expired clientIPAndPort=172.17.0.2:53392, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:13,507 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567412 expired clientIPAndPort=172.17.0.2:53392, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:13,799 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:21:13,800 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f313d7cca28bccec72c3e8e677b94d37, disabling compactions & flushes 2024-11-08T06:21:13,800 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:13,800 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:13,800 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. after waiting 0 ms 2024-11-08T06:21:13,800 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:13,800 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:13,800 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f313d7cca28bccec72c3e8e677b94d37: 2024-11-08T06:21:13,801 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:21:13,801 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731046873801"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046873801"}]},"ts":"1731046873801"} 2024-11-08T06:21:13,802 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:21:13,803 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:21:13,803 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046873803"}]},"ts":"1731046873803"} 2024-11-08T06:21:13,804 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:21:13,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f313d7cca28bccec72c3e8e677b94d37, ASSIGN}] 2024-11-08T06:21:13,808 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f313d7cca28bccec72c3e8e677b94d37, ASSIGN 2024-11-08T06:21:13,809 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f313d7cca28bccec72c3e8e677b94d37, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:21:13,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=f313d7cca28bccec72c3e8e677b94d37, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:13,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure f313d7cca28bccec72c3e8e677b94d37, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:21:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T06:21:14,112 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:14,118 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:14,118 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => f313d7cca28bccec72c3e8e677b94d37, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:21:14,118 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:14,118 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:21:14,118 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:14,118 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:14,120 INFO [StoreOpener-f313d7cca28bccec72c3e8e677b94d37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:14,121 INFO [StoreOpener-f313d7cca28bccec72c3e8e677b94d37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f313d7cca28bccec72c3e8e677b94d37 columnFamilyName testFamily 2024-11-08T06:21:14,121 DEBUG [StoreOpener-f313d7cca28bccec72c3e8e677b94d37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:21:14,122 INFO [StoreOpener-f313d7cca28bccec72c3e8e677b94d37-1 {}] regionserver.HStore(327): Store=f313d7cca28bccec72c3e8e677b94d37/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:21:14,122 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:14,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:14,125 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:14,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:21:14,127 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened f313d7cca28bccec72c3e8e677b94d37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68668231, jitterRate=0.023236379027366638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:21:14,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for f313d7cca28bccec72c3e8e677b94d37: 2024-11-08T06:21:14,128 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37., pid=89, masterSystemTime=1731046874112 2024-11-08T06:21:14,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:14,130 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:14,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=f313d7cca28bccec72c3e8e677b94d37, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:14,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-08T06:21:14,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure f313d7cca28bccec72c3e8e677b94d37, server=d1f4bb6aa14f,36803,1731046638998 in 170 msec 2024-11-08T06:21:14,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-08T06:21:14,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=f313d7cca28bccec72c3e8e677b94d37, ASSIGN in 324 msec 2024-11-08T06:21:14,134 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:21:14,134 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046874134"}]},"ts":"1731046874134"} 2024-11-08T06:21:14,135 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:21:14,137 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:21:14,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1580 sec 2024-11-08T06:21:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T06:21:15,984 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-08T06:21:15,984 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:21:15,985 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:21:15,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-08T06:21:15,988 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567456 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:21:15,989 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-08T06:21:15,990 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:21:15,990 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-08T06:21:15,991 DEBUG [hconnection-0x2fc4ac0d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:21:15,993 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41182, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:21:15,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:21:16,001 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T06:21:16,003 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T06:21:16,004 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567459 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:21:16,005 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-08T06:21:16,006 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567459 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:21:16,006 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-08T06:21:16,303 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-08T06:21:17,403 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T06:21:17,466 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:21:18,611 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567419 expired clientIPAndPort=172.17.0.2:53408, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:21:18,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-08T06:21:18,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:21:19,654 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:21:23,114 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567424 expired clientIPAndPort=172.17.0.2:57724, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:21:25,009 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10013: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10013: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-08T06:21:25,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9004 2024-11-08T06:21:25,010 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2111251f to 127.0.0.1:49997 2024-11-08T06:21:25,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:25,010 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:21:25,010 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:21:25,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:25,010 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7268f750 to 127.0.0.1:49997 2024-11-08T06:21:25,019 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=226 (was 226), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=85 (was 100), ProcessCount=9 (was 9), AvailableMemoryMB=4209 (was 4212) 2024-11-08T06:21:25,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=226, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=85, ProcessCount=9, AvailableMemoryMB=4209 2024-11-08T06:21:25,028 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63a97306 to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c75e020 2024-11-08T06:21:25,031 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x63a97306 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bbbb9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:21:25,031 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x453da24e to 127.0.0.1:49997 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1239134e 2024-11-08T06:21:25,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa0734c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T06:21:25,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T06:21:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T06:21:25,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T06:21:25,038 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:21:25,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-08T06:21:25,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T06:21:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T06:21:25,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741891_1067 (size=431) 2024-11-08T06:21:25,116 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567426 expired clientIPAndPort=172.17.0.2:57724, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:21:25,452 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1bb1e1befa856ff8a272061f95c2a2d0, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a 2024-11-08T06:21:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741892_1068 (size=82) 2024-11-08T06:21:25,506 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:21:25,859 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:21:25,859 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 1bb1e1befa856ff8a272061f95c2a2d0, disabling compactions & flushes 2024-11-08T06:21:25,859 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:25,859 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:25,859 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. after waiting 0 ms 2024-11-08T06:21:25,859 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:25,859 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:25,859 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1bb1e1befa856ff8a272061f95c2a2d0: 2024-11-08T06:21:25,860 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T06:21:25,860 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731046885860"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731046885860"}]},"ts":"1731046885860"} 2024-11-08T06:21:25,862 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T06:21:25,863 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T06:21:25,863 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046885863"}]},"ts":"1731046885863"} 2024-11-08T06:21:25,864 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-08T06:21:25,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1bb1e1befa856ff8a272061f95c2a2d0, ASSIGN}] 2024-11-08T06:21:25,868 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1bb1e1befa856ff8a272061f95c2a2d0, ASSIGN 2024-11-08T06:21:25,869 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1bb1e1befa856ff8a272061f95c2a2d0, ASSIGN; state=OFFLINE, location=d1f4bb6aa14f,36803,1731046638998; forceNewPlan=false, retain=false 2024-11-08T06:21:26,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=1bb1e1befa856ff8a272061f95c2a2d0, regionState=OPENING, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:26,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 1bb1e1befa856ff8a272061f95c2a2d0, server=d1f4bb6aa14f,36803,1731046638998}] 2024-11-08T06:21:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T06:21:26,172 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:26,177 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:26,177 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 1bb1e1befa856ff8a272061f95c2a2d0, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T06:21:26,178 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:26,178 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T06:21:26,178 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:26,178 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:26,179 INFO [StoreOpener-1bb1e1befa856ff8a272061f95c2a2d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:26,181 INFO [StoreOpener-1bb1e1befa856ff8a272061f95c2a2d0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bb1e1befa856ff8a272061f95c2a2d0 columnFamilyName testFamily 2024-11-08T06:21:26,181 DEBUG [StoreOpener-1bb1e1befa856ff8a272061f95c2a2d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T06:21:26,182 INFO [StoreOpener-1bb1e1befa856ff8a272061f95c2a2d0-1 {}] regionserver.HStore(327): Store=1bb1e1befa856ff8a272061f95c2a2d0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T06:21:26,183 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:26,183 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:26,185 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:26,187 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T06:21:26,188 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 1bb1e1befa856ff8a272061f95c2a2d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59300515, jitterRate=-0.11635346710681915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T06:21:26,188 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 1bb1e1befa856ff8a272061f95c2a2d0: 2024-11-08T06:21:26,189 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., pid=92, masterSystemTime=1731046886172 2024-11-08T06:21:26,190 DEBUG [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:26,190 INFO [RS_OPEN_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:26,191 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=1bb1e1befa856ff8a272061f95c2a2d0, regionState=OPEN, openSeqNum=2, regionLocation=d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:26,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-08T06:21:26,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 1bb1e1befa856ff8a272061f95c2a2d0, server=d1f4bb6aa14f,36803,1731046638998 in 171 msec 2024-11-08T06:21:26,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-08T06:21:26,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=1bb1e1befa856ff8a272061f95c2a2d0, ASSIGN in 326 msec 2024-11-08T06:21:26,195 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T06:21:26,195 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731046886195"}]},"ts":"1731046886195"} 2024-11-08T06:21:26,196 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-08T06:21:26,199 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T06:21:26,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1630 sec 2024-11-08T06:21:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46003 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-08T06:21:28,042 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-08T06:21:28,042 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-08T06:21:28,042 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:21:28,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36803 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-08T06:21:28,046 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 8767934147168567460 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-08T06:21:28,047 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-08T06:21:28,047 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T06:21:28,047 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-08T06:21:28,049 DEBUG [hconnection-0x74ed87fe-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:21:28,050 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60030, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:21:28,057 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-08T06:21:28,059 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T06:21:28,059 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-08T06:21:28,060 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x63a97306 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=-1] 2024-11-08T06:21:28,060 DEBUG [ReadOnlyZKClient-127.0.0.1:49997@0x63a97306 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T06:21:28,061 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60034, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T06:21:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:21:28,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:21:28,305 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-08T06:21:28,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T06:21:28,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-08T06:21:29,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-08T06:21:29,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-08T06:21:29,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:21:29,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 88 bytes 2024-11-08T06:21:30,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2] 2024-11-08T06:21:30,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36803 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-08T06:21:31,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=975ms 2024-11-08T06:21:31,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=975ms 2024-11-08T06:21:31,068 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0., hostname=d1f4bb6aa14f,36803,1731046638998, seqNum=2 from cache 2024-11-08T06:21:31,069 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-08T06:21:29.068Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-08T06:21:31.068Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=975ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d1f4bb6aa14f:36803 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=985ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-08T06:21:31,070 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3010 2024-11-08T06:21:31,070 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x453da24e to 127.0.0.1:49997 2024-11-08T06:21:31,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:31,070 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-08T06:21:31,070 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor86.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T06:21:31,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:31,070 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63a97306 to 127.0.0.1:49997 2024-11-08T06:21:31,079 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=230 (was 226) - Thread LEAK? -, OpenFileDescriptor=449 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=78 (was 85), ProcessCount=9 (was 9), AvailableMemoryMB=4206 (was 4209) 2024-11-08T06:21:31,079 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-08T06:21:31,079 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T06:21:31,079 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x574a453f to 127.0.0.1:49997 2024-11-08T06:21:31,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:31,079 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T06:21:31,079 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1501590834, stopped=false 2024-11-08T06:21:31,080 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:21:31,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T06:21:31,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T06:21:31,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:21:31,082 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-08T06:21:31,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:21:31,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:31,082 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd1f4bb6aa14f,36803,1731046638998' ***** 2024-11-08T06:21:31,082 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-08T06:21:31,082 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T06:21:31,082 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T06:21:31,083 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T06:21:31,083 INFO [RS:0;d1f4bb6aa14f:36803 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T06:21:31,083 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-08T06:21:31,083 INFO [RS:0;d1f4bb6aa14f:36803 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T06:21:31,083 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for f33aa95219c15aed37fc2fd9cd667ba3 2024-11-08T06:21:31,083 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:21:31,083 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for e8e8dada2468fd2a22641e1cf25b4ac0 2024-11-08T06:21:31,083 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for bef1d5c4b90cf2bf782264d6131b3426 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 4452a1cba67ce3322ca17c876abcd582 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 506c2ac65720fba05f47cd2e4def3562 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 4f990b745771e77a5b0a8093dac6553c 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for a45c97b247e9558943305207041e58d5 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 3dfc4ceec119f4060839de57ca17c059 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for e2211a6e8e961230cc6dbdd2a2186983 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 169fe32fc4194690caf6575dbee1d411 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 2cb28dcad64759a3f944589ee2bafd95 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 52eec85d1f18d9e6fb9f54d6a6190ed7 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for a99ec1db5991df8c4fc52125fca5b6cb 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 75cd2864f9d52d49cb12429ecdc3da83 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 40465c665e78ed129aec58b4cd8a7ce6 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 13de1ea4d8dd655db7256497e2b35cd6 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 58b3ddfd049d1caa4403adb146662c75 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for ea36b5332d3c79bdc64235d12d2c6bf0 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 640f50ffb597620ddc4800f241b7ca8f 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 1052053f1be9a398e9e6217d04d32d31 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 8443cee1b4a7317c15452e64a36df3da 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for e60b79623b21cdaf896284976a0739d4 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 0b865dba842e12e627d69ce50c784684 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 79c06b38371ce93f62765ff1ce21f968 2024-11-08T06:21:31,084 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f33aa95219c15aed37fc2fd9cd667ba3, disabling compactions & flushes 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 50376f508b5217860f26c5f8a8fa984c 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:31,084 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:31,084 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1224): stopping server d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:31,084 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. after waiting 0 ms 2024-11-08T06:21:31,084 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:31,084 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T06:21:31,084 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-08T06:21:31,085 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-08T06:21:31,085 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f33aa95219c15aed37fc2fd9cd667ba3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:31,085 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T06:21:31,085 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T06:21:31,085 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T06:21:31,085 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T06:21:31,085 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T06:21:31,085 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-08T06:21:31,085 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1588230740, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37, f33aa95219c15aed37fc2fd9cd667ba3, faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:21:31,141 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3/.tmp/testFamily/066974bbb92d48e08f4e3e20050371b7 is 61, key is row-0/testFamily:testQualifier/1731046813425/Put/seqid=0 2024-11-08T06:21:31,151 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/.tmp/info/3e79aece377648fb9108eb7a5e6fda9a is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582./info:regioninfo/1731046704675/Put/seqid=0 2024-11-08T06:21:31,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741893_1069 (size=5049) 2024-11-08T06:21:31,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741894_1070 (size=51873) 2024-11-08T06:21:31,220 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease 8767934147168567431 expired clientIPAndPort=172.17.0.2:57750, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:21:31,285 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1588230740, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37, f33aa95219c15aed37fc2fd9cd667ba3, faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:21:31,486 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1588230740, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37, f33aa95219c15aed37fc2fd9cd667ba3, faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:21:31,510 INFO [regionserver/d1f4bb6aa14f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T06:21:31,510 INFO [regionserver/d1f4bb6aa14f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T06:21:31,552 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3/.tmp/testFamily/066974bbb92d48e08f4e3e20050371b7 2024-11-08T06:21:31,556 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/.tmp/info/3e79aece377648fb9108eb7a5e6fda9a 2024-11-08T06:21:31,584 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T06:21:31,596 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3/.tmp/testFamily/066974bbb92d48e08f4e3e20050371b7 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3/testFamily/066974bbb92d48e08f4e3e20050371b7 2024-11-08T06:21:31,604 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3/testFamily/066974bbb92d48e08f4e3e20050371b7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:31,606 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f33aa95219c15aed37fc2fd9cd667ba3 in 521ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:31,611 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_1_-true/f33aa95219c15aed37fc2fd9cd667ba3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:31,614 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:31,614 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f33aa95219c15aed37fc2fd9cd667ba3: 2024-11-08T06:21:31,614 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731046810409.f33aa95219c15aed37fc2fd9cd667ba3. 2024-11-08T06:21:31,614 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing faef91a41733387bbcd9cb53b34883eb, disabling compactions & flushes 2024-11-08T06:21:31,614 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:21:31,614 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:21:31,614 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. after waiting 0 ms 2024-11-08T06:21:31,614 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:21:31,614 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing faef91a41733387bbcd9cb53b34883eb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:31,615 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/.tmp/table/a7cba3a93acf45069ecfb4d41ffda85e is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1731046704687/Put/seqid=0 2024-11-08T06:21:31,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741895_1071 (size=9821) 2024-11-08T06:21:31,630 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb/.tmp/testFamily/4ac7926ab7fa4b8dba409c513426af73 is 61, key is row-0/testFamily:testQualifier/1731046847799/Put/seqid=0 2024-11-08T06:21:31,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741896_1072 (size=5049) 2024-11-08T06:21:31,686 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1588230740, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37, faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:21:31,713 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:21:31,886 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1588230740, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37, faef91a41733387bbcd9cb53b34883eb 2024-11-08T06:21:32,020 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/.tmp/table/a7cba3a93acf45069ecfb4d41ffda85e 2024-11-08T06:21:32,028 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/.tmp/info/3e79aece377648fb9108eb7a5e6fda9a as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/info/3e79aece377648fb9108eb7a5e6fda9a 2024-11-08T06:21:32,034 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb/.tmp/testFamily/4ac7926ab7fa4b8dba409c513426af73 2024-11-08T06:21:32,034 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/info/3e79aece377648fb9108eb7a5e6fda9a, entries=290, sequenceid=149, filesize=50.7 K 2024-11-08T06:21:32,036 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/.tmp/table/a7cba3a93acf45069ecfb4d41ffda85e as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/table/a7cba3a93acf45069ecfb4d41ffda85e 2024-11-08T06:21:32,041 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb/.tmp/testFamily/4ac7926ab7fa4b8dba409c513426af73 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb/testFamily/4ac7926ab7fa4b8dba409c513426af73 2024-11-08T06:21:32,043 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/table/a7cba3a93acf45069ecfb4d41ffda85e, entries=58, sequenceid=149, filesize=9.6 K 2024-11-08T06:21:32,044 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 959ms, sequenceid=149, compaction requested=false 2024-11-08T06:21:32,048 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb/testFamily/4ac7926ab7fa4b8dba409c513426af73, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:32,049 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-08T06:21:32,050 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T06:21:32,050 INFO [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T06:21:32,050 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T06:21:32,050 DEBUG [RS_CLOSE_META-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T06:21:32,051 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for faef91a41733387bbcd9cb53b34883eb in 437ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:32,055 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_1_-true/faef91a41733387bbcd9cb53b34883eb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:32,056 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:21:32,056 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for faef91a41733387bbcd9cb53b34883eb: 2024-11-08T06:21:32,056 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731046844783.faef91a41733387bbcd9cb53b34883eb. 2024-11-08T06:21:32,056 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e8e8dada2468fd2a22641e1cf25b4ac0, disabling compactions & flushes 2024-11-08T06:21:32,056 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:21:32,056 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:21:32,056 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. after waiting 0 ms 2024-11-08T06:21:32,056 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:21:32,056 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e8e8dada2468fd2a22641e1cf25b4ac0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:32,072 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0/.tmp/testFamily/df05fbeb536345abb54fd6dbb15dff2a is 61, key is row-0/testFamily:testQualifier/1731046788228/Put/seqid=0 2024-11-08T06:21:32,073 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:45761,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:21:32,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741897_1073 (size=5049) 2024-11-08T06:21:32,087 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-08T06:21:32,087 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:32,287 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:32,477 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0/.tmp/testFamily/df05fbeb536345abb54fd6dbb15dff2a 2024-11-08T06:21:32,485 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0/.tmp/testFamily/df05fbeb536345abb54fd6dbb15dff2a as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0/testFamily/df05fbeb536345abb54fd6dbb15dff2a 2024-11-08T06:21:32,487 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, e8e8dada2468fd2a22641e1cf25b4ac0, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:32,491 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0/testFamily/df05fbeb536345abb54fd6dbb15dff2a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:32,491 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e8e8dada2468fd2a22641e1cf25b4ac0 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:32,495 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/e8e8dada2468fd2a22641e1cf25b4ac0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:32,496 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:21:32,496 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e8e8dada2468fd2a22641e1cf25b4ac0: 2024-11-08T06:21:32,496 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731046787211.e8e8dada2468fd2a22641e1cf25b4ac0. 2024-11-08T06:21:32,496 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing bef1d5c4b90cf2bf782264d6131b3426, disabling compactions & flushes 2024-11-08T06:21:32,496 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:21:32,496 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:21:32,496 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. after waiting 0 ms 2024-11-08T06:21:32,496 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:21:32,496 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing bef1d5c4b90cf2bf782264d6131b3426 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:32,512 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426/.tmp/testFamily/8a96ffdb39684eee8ab8a3cd0b1f7d62 is 61, key is row-0/testFamily:testQualifier/1731046708044/Put/seqid=0 2024-11-08T06:21:32,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741898_1074 (size=5049) 2024-11-08T06:21:32,687 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:32,888 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, bef1d5c4b90cf2bf782264d6131b3426, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:32,916 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426/.tmp/testFamily/8a96ffdb39684eee8ab8a3cd0b1f7d62 2024-11-08T06:21:32,924 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426/.tmp/testFamily/8a96ffdb39684eee8ab8a3cd0b1f7d62 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426/testFamily/8a96ffdb39684eee8ab8a3cd0b1f7d62 2024-11-08T06:21:32,930 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426/testFamily/8a96ffdb39684eee8ab8a3cd0b1f7d62, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:32,931 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bef1d5c4b90cf2bf782264d6131b3426 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:32,935 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/bef1d5c4b90cf2bf782264d6131b3426/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:32,936 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:21:32,936 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for bef1d5c4b90cf2bf782264d6131b3426: 2024-11-08T06:21:32,936 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731046705015.bef1d5c4b90cf2bf782264d6131b3426. 2024-11-08T06:21:32,936 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4452a1cba67ce3322ca17c876abcd582, disabling compactions & flushes 2024-11-08T06:21:32,936 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:21:32,936 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:21:32,936 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. after waiting 0 ms 2024-11-08T06:21:32,936 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:21:32,936 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4452a1cba67ce3322ca17c876abcd582 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:32,952 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582/.tmp/testFamily/0c385bc0c8e34e6188b5ae9559c645f3 is 61, key is row-0/testFamily:testQualifier/1731046704916/Put/seqid=0 2024-11-08T06:21:32,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741899_1075 (size=5049) 2024-11-08T06:21:33,088 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 25 regions to close 2024-11-08T06:21:33,088 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:33,288 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4452a1cba67ce3322ca17c876abcd582, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:33,356 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582/.tmp/testFamily/0c385bc0c8e34e6188b5ae9559c645f3 2024-11-08T06:21:33,364 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582/.tmp/testFamily/0c385bc0c8e34e6188b5ae9559c645f3 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582/testFamily/0c385bc0c8e34e6188b5ae9559c645f3 2024-11-08T06:21:33,370 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582/testFamily/0c385bc0c8e34e6188b5ae9559c645f3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:33,371 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4452a1cba67ce3322ca17c876abcd582 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:33,375 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/4452a1cba67ce3322ca17c876abcd582/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:33,376 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:21:33,376 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4452a1cba67ce3322ca17c876abcd582: 2024-11-08T06:21:33,376 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731046703882.4452a1cba67ce3322ca17c876abcd582. 2024-11-08T06:21:33,376 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 506c2ac65720fba05f47cd2e4def3562, disabling compactions & flushes 2024-11-08T06:21:33,376 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:21:33,376 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:21:33,376 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. after waiting 0 ms 2024-11-08T06:21:33,376 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:21:33,376 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 506c2ac65720fba05f47cd2e4def3562 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:33,392 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562/.tmp/testFamily/8e96c712b7834e0e839ca824a75b06ac is 61, key is row-0/testFamily:testQualifier/1731046752815/Put/seqid=0 2024-11-08T06:21:33,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741900_1076 (size=5049) 2024-11-08T06:21:33,489 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:33,689 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 506c2ac65720fba05f47cd2e4def3562, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:33,797 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562/.tmp/testFamily/8e96c712b7834e0e839ca824a75b06ac 2024-11-08T06:21:33,805 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562/.tmp/testFamily/8e96c712b7834e0e839ca824a75b06ac as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562/testFamily/8e96c712b7834e0e839ca824a75b06ac 2024-11-08T06:21:33,811 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562/testFamily/8e96c712b7834e0e839ca824a75b06ac, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:33,812 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 506c2ac65720fba05f47cd2e4def3562 in 436ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:33,815 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNext_0_-false/506c2ac65720fba05f47cd2e4def3562/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:33,816 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:21:33,816 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 506c2ac65720fba05f47cd2e4def3562: 2024-11-08T06:21:33,816 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1731046751793.506c2ac65720fba05f47cd2e4def3562. 2024-11-08T06:21:33,816 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4f990b745771e77a5b0a8093dac6553c, disabling compactions & flushes 2024-11-08T06:21:33,816 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:21:33,816 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:21:33,816 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. after waiting 0 ms 2024-11-08T06:21:33,816 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:21:33,816 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4f990b745771e77a5b0a8093dac6553c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:33,832 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c/.tmp/testFamily/fd24b232e0a24ce1b805461ccc982066 is 61, key is row-0/testFamily:testQualifier/1731046718149/Put/seqid=0 2024-11-08T06:21:33,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741901_1077 (size=5049) 2024-11-08T06:21:33,889 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:34,089 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 23 regions to close 2024-11-08T06:21:34,089 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 4f990b745771e77a5b0a8093dac6553c, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:34,237 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c/.tmp/testFamily/fd24b232e0a24ce1b805461ccc982066 2024-11-08T06:21:34,245 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c/.tmp/testFamily/fd24b232e0a24ce1b805461ccc982066 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c/testFamily/fd24b232e0a24ce1b805461ccc982066 2024-11-08T06:21:34,251 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c/testFamily/fd24b232e0a24ce1b805461ccc982066, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:34,252 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4f990b745771e77a5b0a8093dac6553c in 436ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:34,255 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4f990b745771e77a5b0a8093dac6553c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:34,256 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:21:34,256 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4f990b745771e77a5b0a8093dac6553c: 2024-11-08T06:21:34,256 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731046717124.4f990b745771e77a5b0a8093dac6553c. 2024-11-08T06:21:34,256 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a45c97b247e9558943305207041e58d5, disabling compactions & flushes 2024-11-08T06:21:34,256 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:21:34,256 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:21:34,256 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. after waiting 0 ms 2024-11-08T06:21:34,256 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:21:34,256 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a45c97b247e9558943305207041e58d5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:34,272 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5/.tmp/testFamily/57ea0a5d6b9f4a33987241f82a8ca463 is 61, key is row-0/testFamily:testQualifier/1731046853863/Put/seqid=0 2024-11-08T06:21:34,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741902_1078 (size=5049) 2024-11-08T06:21:34,290 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:34,490 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:34,676 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5/.tmp/testFamily/57ea0a5d6b9f4a33987241f82a8ca463 2024-11-08T06:21:34,685 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5/.tmp/testFamily/57ea0a5d6b9f4a33987241f82a8ca463 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5/testFamily/57ea0a5d6b9f4a33987241f82a8ca463 2024-11-08T06:21:34,690 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a45c97b247e9558943305207041e58d5, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:34,691 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5/testFamily/57ea0a5d6b9f4a33987241f82a8ca463, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:34,692 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a45c97b247e9558943305207041e58d5 in 436ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:34,696 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/a45c97b247e9558943305207041e58d5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:34,697 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:21:34,697 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a45c97b247e9558943305207041e58d5: 2024-11-08T06:21:34,697 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731046850846.a45c97b247e9558943305207041e58d5. 2024-11-08T06:21:34,697 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3dfc4ceec119f4060839de57ca17c059, disabling compactions & flushes 2024-11-08T06:21:34,697 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:21:34,697 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:21:34,697 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. after waiting 0 ms 2024-11-08T06:21:34,697 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:21:34,697 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3dfc4ceec119f4060839de57ca17c059 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:34,713 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059/.tmp/testFamily/48370e94f9db45c491a9ae1a87610031 is 61, key is row-0/testFamily:testQualifier/1731046843246/Put/seqid=0 2024-11-08T06:21:34,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741903_1079 (size=5049) 2024-11-08T06:21:34,890 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:35,090 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-08T06:21:35,090 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 3dfc4ceec119f4060839de57ca17c059, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:35,117 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059/.tmp/testFamily/48370e94f9db45c491a9ae1a87610031 2024-11-08T06:21:35,124 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059/.tmp/testFamily/48370e94f9db45c491a9ae1a87610031 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059/testFamily/48370e94f9db45c491a9ae1a87610031 2024-11-08T06:21:35,130 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059/testFamily/48370e94f9db45c491a9ae1a87610031, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:35,131 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3dfc4ceec119f4060839de57ca17c059 in 434ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:35,135 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_1_-true/3dfc4ceec119f4060839de57ca17c059/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:35,135 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:21:35,135 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3dfc4ceec119f4060839de57ca17c059: 2024-11-08T06:21:35,136 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731046840229.3dfc4ceec119f4060839de57ca17c059. 2024-11-08T06:21:35,136 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e2211a6e8e961230cc6dbdd2a2186983, disabling compactions & flushes 2024-11-08T06:21:35,136 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:21:35,136 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:21:35,136 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. after waiting 0 ms 2024-11-08T06:21:35,136 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:21:35,136 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e2211a6e8e961230cc6dbdd2a2186983 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:35,152 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983/.tmp/testFamily/d31a4f380a6a4aa1bbdf2b7b1500cd97 is 61, key is row-0/testFamily:testQualifier/1731046729306/Put/seqid=0 2024-11-08T06:21:35,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741904_1080 (size=5049) 2024-11-08T06:21:35,291 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:35,491 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e2211a6e8e961230cc6dbdd2a2186983, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:35,556 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983/.tmp/testFamily/d31a4f380a6a4aa1bbdf2b7b1500cd97 2024-11-08T06:21:35,563 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983/.tmp/testFamily/d31a4f380a6a4aa1bbdf2b7b1500cd97 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983/testFamily/d31a4f380a6a4aa1bbdf2b7b1500cd97 2024-11-08T06:21:35,569 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983/testFamily/d31a4f380a6a4aa1bbdf2b7b1500cd97, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:35,570 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e2211a6e8e961230cc6dbdd2a2186983 in 434ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:35,573 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_0_-false/e2211a6e8e961230cc6dbdd2a2186983/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:35,574 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:21:35,574 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e2211a6e8e961230cc6dbdd2a2186983: 2024-11-08T06:21:35,574 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731046728288.e2211a6e8e961230cc6dbdd2a2186983. 2024-11-08T06:21:35,574 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 169fe32fc4194690caf6575dbee1d411, disabling compactions & flushes 2024-11-08T06:21:35,574 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:21:35,574 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:21:35,574 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. after waiting 0 ms 2024-11-08T06:21:35,574 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:21:35,574 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 169fe32fc4194690caf6575dbee1d411 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:35,590 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411/.tmp/testFamily/337adfba7b624cbdbd0920e19b19f97c is 61, key is row-0/testFamily:testQualifier/1731046738489/Put/seqid=0 2024-11-08T06:21:35,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741905_1081 (size=5049) 2024-11-08T06:21:35,691 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:35,891 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 169fe32fc4194690caf6575dbee1d411, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:35,994 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411/.tmp/testFamily/337adfba7b624cbdbd0920e19b19f97c 2024-11-08T06:21:36,002 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411/.tmp/testFamily/337adfba7b624cbdbd0920e19b19f97c as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411/testFamily/337adfba7b624cbdbd0920e19b19f97c 2024-11-08T06:21:36,008 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411/testFamily/337adfba7b624cbdbd0920e19b19f97c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:36,009 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 169fe32fc4194690caf6575dbee1d411 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:36,013 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_0_-false/169fe32fc4194690caf6575dbee1d411/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:36,013 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:21:36,013 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 169fe32fc4194690caf6575dbee1d411: 2024-11-08T06:21:36,013 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731046735453.169fe32fc4194690caf6575dbee1d411. 2024-11-08T06:21:36,014 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2cb28dcad64759a3f944589ee2bafd95, disabling compactions & flushes 2024-11-08T06:21:36,014 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:21:36,014 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:21:36,014 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. after waiting 0 ms 2024-11-08T06:21:36,014 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:21:36,014 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2cb28dcad64759a3f944589ee2bafd95 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:36,030 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95/.tmp/testFamily/ac046ac81bf144f3bfe249f0be861cd5 is 61, key is row-0/testFamily:testQualifier/1731046732402/Put/seqid=0 2024-11-08T06:21:36,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741906_1082 (size=5049) 2024-11-08T06:21:36,092 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 18 regions to close 2024-11-08T06:21:36,092 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:36,292 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 2cb28dcad64759a3f944589ee2bafd95, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:36,435 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95/.tmp/testFamily/ac046ac81bf144f3bfe249f0be861cd5 2024-11-08T06:21:36,442 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95/.tmp/testFamily/ac046ac81bf144f3bfe249f0be861cd5 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95/testFamily/ac046ac81bf144f3bfe249f0be861cd5 2024-11-08T06:21:36,448 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95/testFamily/ac046ac81bf144f3bfe249f0be861cd5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:36,449 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2cb28dcad64759a3f944589ee2bafd95 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:36,453 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/2cb28dcad64759a3f944589ee2bafd95/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:36,453 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:21:36,453 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2cb28dcad64759a3f944589ee2bafd95: 2024-11-08T06:21:36,453 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731046731382.2cb28dcad64759a3f944589ee2bafd95. 2024-11-08T06:21:36,453 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 52eec85d1f18d9e6fb9f54d6a6190ed7, disabling compactions & flushes 2024-11-08T06:21:36,453 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:21:36,454 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:21:36,454 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. after waiting 0 ms 2024-11-08T06:21:36,454 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:21:36,454 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 52eec85d1f18d9e6fb9f54d6a6190ed7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:36,469 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7/.tmp/testFamily/fa2e3bca898c43c0a26acb7977fb0652 is 61, key is row-0/testFamily:testQualifier/1731046833692/Put/seqid=0 2024-11-08T06:21:36,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741907_1083 (size=5049) 2024-11-08T06:21:36,492 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:36,692 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 52eec85d1f18d9e6fb9f54d6a6190ed7, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:36,873 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7/.tmp/testFamily/fa2e3bca898c43c0a26acb7977fb0652 2024-11-08T06:21:36,881 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7/.tmp/testFamily/fa2e3bca898c43c0a26acb7977fb0652 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7/testFamily/fa2e3bca898c43c0a26acb7977fb0652 2024-11-08T06:21:36,886 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7/testFamily/fa2e3bca898c43c0a26acb7977fb0652, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:36,887 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 52eec85d1f18d9e6fb9f54d6a6190ed7 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:36,891 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_1_-true/52eec85d1f18d9e6fb9f54d6a6190ed7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:36,891 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:21:36,892 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 52eec85d1f18d9e6fb9f54d6a6190ed7: 2024-11-08T06:21:36,892 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731046830671.52eec85d1f18d9e6fb9f54d6a6190ed7. 2024-11-08T06:21:36,892 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing a99ec1db5991df8c4fc52125fca5b6cb, disabling compactions & flushes 2024-11-08T06:21:36,892 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:36,892 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:36,892 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. after waiting 0 ms 2024-11-08T06:21:36,892 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:36,892 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing a99ec1db5991df8c4fc52125fca5b6cb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:36,892 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:36,907 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb/.tmp/testFamily/5fc4150ab8b9478498589367151bd52a is 61, key is row-0/testFamily:testQualifier/1731046863937/Put/seqid=0 2024-11-08T06:21:36,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741908_1084 (size=5049) 2024-11-08T06:21:37,093 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 16 regions to close 2024-11-08T06:21:37,093 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:37,115 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:21:37,293 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, a99ec1db5991df8c4fc52125fca5b6cb, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:37,312 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb/.tmp/testFamily/5fc4150ab8b9478498589367151bd52a 2024-11-08T06:21:37,319 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb/.tmp/testFamily/5fc4150ab8b9478498589367151bd52a as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb/testFamily/5fc4150ab8b9478498589367151bd52a 2024-11-08T06:21:37,325 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb/testFamily/5fc4150ab8b9478498589367151bd52a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:37,326 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a99ec1db5991df8c4fc52125fca5b6cb in 434ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:37,330 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a99ec1db5991df8c4fc52125fca5b6cb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:37,331 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:37,331 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for a99ec1db5991df8c4fc52125fca5b6cb: 2024-11-08T06:21:37,331 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731046860921.a99ec1db5991df8c4fc52125fca5b6cb. 2024-11-08T06:21:37,331 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 75cd2864f9d52d49cb12429ecdc3da83, disabling compactions & flushes 2024-11-08T06:21:37,331 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:21:37,331 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:21:37,331 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. after waiting 0 ms 2024-11-08T06:21:37,331 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:21:37,331 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 75cd2864f9d52d49cb12429ecdc3da83 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:37,347 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83/.tmp/testFamily/964998c384cb42ea8f6207d0b4e95cd5 is 61, key is row-0/testFamily:testQualifier/1731046750222/Put/seqid=0 2024-11-08T06:21:37,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741909_1085 (size=5049) 2024-11-08T06:21:37,493 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:37,693 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 75cd2864f9d52d49cb12429ecdc3da83, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:37,752 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83/.tmp/testFamily/964998c384cb42ea8f6207d0b4e95cd5 2024-11-08T06:21:37,758 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83/.tmp/testFamily/964998c384cb42ea8f6207d0b4e95cd5 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83/testFamily/964998c384cb42ea8f6207d0b4e95cd5 2024-11-08T06:21:37,764 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83/testFamily/964998c384cb42ea8f6207d0b4e95cd5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:37,764 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 75cd2864f9d52d49cb12429ecdc3da83 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:37,768 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnClose_0_-false/75cd2864f9d52d49cb12429ecdc3da83/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:37,769 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:21:37,769 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 75cd2864f9d52d49cb12429ecdc3da83: 2024-11-08T06:21:37,769 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1731046749199.75cd2864f9d52d49cb12429ecdc3da83. 2024-11-08T06:21:37,769 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 40465c665e78ed129aec58b4cd8a7ce6, disabling compactions & flushes 2024-11-08T06:21:37,769 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:21:37,769 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:21:37,769 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. after waiting 0 ms 2024-11-08T06:21:37,769 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:21:37,770 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 40465c665e78ed129aec58b4cd8a7ce6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:37,785 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6/.tmp/testFamily/a82895d05df64a95891736b176bea288 is 61, key is row-0/testFamily:testQualifier/1731046791303/Put/seqid=0 2024-11-08T06:21:37,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741910_1086 (size=5049) 2024-11-08T06:21:37,894 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:38,094 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 14 regions to close 2024-11-08T06:21:38,094 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 40465c665e78ed129aec58b4cd8a7ce6, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:38,190 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6/.tmp/testFamily/a82895d05df64a95891736b176bea288 2024-11-08T06:21:38,197 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6/.tmp/testFamily/a82895d05df64a95891736b176bea288 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6/testFamily/a82895d05df64a95891736b176bea288 2024-11-08T06:21:38,203 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6/testFamily/a82895d05df64a95891736b176bea288, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:38,204 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 40465c665e78ed129aec58b4cd8a7ce6 in 435ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:38,208 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/40465c665e78ed129aec58b4cd8a7ce6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:38,208 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:21:38,209 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 40465c665e78ed129aec58b4cd8a7ce6: 2024-11-08T06:21:38,209 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731046788284.40465c665e78ed129aec58b4cd8a7ce6. 2024-11-08T06:21:38,209 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 13de1ea4d8dd655db7256497e2b35cd6, disabling compactions & flushes 2024-11-08T06:21:38,209 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:21:38,209 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:21:38,209 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. after waiting 0 ms 2024-11-08T06:21:38,209 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:21:38,209 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 13de1ea4d8dd655db7256497e2b35cd6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:38,224 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6/.tmp/testFamily/985eb5ee42604adb950ddf14f37e4733 is 61, key is row-0/testFamily:testQualifier/1731046763014/Put/seqid=0 2024-11-08T06:21:38,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741911_1087 (size=5049) 2024-11-08T06:21:38,294 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:38,494 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 13de1ea4d8dd655db7256497e2b35cd6, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:38,629 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6/.tmp/testFamily/985eb5ee42604adb950ddf14f37e4733 2024-11-08T06:21:38,636 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6/.tmp/testFamily/985eb5ee42604adb950ddf14f37e4733 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6/testFamily/985eb5ee42604adb950ddf14f37e4733 2024-11-08T06:21:38,641 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6/testFamily/985eb5ee42604adb950ddf14f37e4733, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:38,642 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 13de1ea4d8dd655db7256497e2b35cd6 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:38,646 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/13de1ea4d8dd655db7256497e2b35cd6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:38,647 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:21:38,647 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 13de1ea4d8dd655db7256497e2b35cd6: 2024-11-08T06:21:38,647 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731046761997.13de1ea4d8dd655db7256497e2b35cd6. 2024-11-08T06:21:38,647 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 58b3ddfd049d1caa4403adb146662c75, disabling compactions & flushes 2024-11-08T06:21:38,647 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:21:38,647 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:21:38,647 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. after waiting 0 ms 2024-11-08T06:21:38,647 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:21:38,647 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 58b3ddfd049d1caa4403adb146662c75 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:38,663 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75/.tmp/testFamily/2d395bac4850442993185c774c6d1a7b is 61, key is row-0/testFamily:testQualifier/1731046754925/Put/seqid=0 2024-11-08T06:21:38,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741912_1088 (size=5049) 2024-11-08T06:21:38,695 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-08T06:21:38,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-08T06:21:38,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T06:21:38,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-08T06:21:38,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-08T06:21:38,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-08T06:21:38,895 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 58b3ddfd049d1caa4403adb146662c75, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:39,068 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75/.tmp/testFamily/2d395bac4850442993185c774c6d1a7b 2024-11-08T06:21:39,075 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75/.tmp/testFamily/2d395bac4850442993185c774c6d1a7b as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75/testFamily/2d395bac4850442993185c774c6d1a7b 2024-11-08T06:21:39,080 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75/testFamily/2d395bac4850442993185c774c6d1a7b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:39,081 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 58b3ddfd049d1caa4403adb146662c75 in 434ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:39,085 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/58b3ddfd049d1caa4403adb146662c75/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:39,085 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:21:39,085 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 58b3ddfd049d1caa4403adb146662c75: 2024-11-08T06:21:39,085 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731046753902.58b3ddfd049d1caa4403adb146662c75. 2024-11-08T06:21:39,086 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ea36b5332d3c79bdc64235d12d2c6bf0, disabling compactions & flushes 2024-11-08T06:21:39,086 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:21:39,086 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:21:39,086 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. after waiting 0 ms 2024-11-08T06:21:39,086 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:21:39,086 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ea36b5332d3c79bdc64235d12d2c6bf0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:39,095 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 11 regions to close 2024-11-08T06:21:39,095 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:39,101 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0/.tmp/testFamily/27632824eb7945c0b864ae8f4f14d43d is 61, key is row-0/testFamily:testQualifier/1731046816495/Put/seqid=0 2024-11-08T06:21:39,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741913_1089 (size=5049) 2024-11-08T06:21:39,295 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:39,495 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, ea36b5332d3c79bdc64235d12d2c6bf0, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:39,505 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0/.tmp/testFamily/27632824eb7945c0b864ae8f4f14d43d 2024-11-08T06:21:39,512 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0/.tmp/testFamily/27632824eb7945c0b864ae8f4f14d43d as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0/testFamily/27632824eb7945c0b864ae8f4f14d43d 2024-11-08T06:21:39,518 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0/testFamily/27632824eb7945c0b864ae8f4f14d43d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:39,518 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ea36b5332d3c79bdc64235d12d2c6bf0 in 432ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:39,522 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnRenewLease_1_-true/ea36b5332d3c79bdc64235d12d2c6bf0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:39,523 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:21:39,523 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ea36b5332d3c79bdc64235d12d2c6bf0: 2024-11-08T06:21:39,523 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731046813478.ea36b5332d3c79bdc64235d12d2c6bf0. 2024-11-08T06:21:39,523 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 640f50ffb597620ddc4800f241b7ca8f, disabling compactions & flushes 2024-11-08T06:21:39,523 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:21:39,523 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:21:39,523 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. after waiting 0 ms 2024-11-08T06:21:39,523 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:21:39,523 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 640f50ffb597620ddc4800f241b7ca8f 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-08T06:21:39,539 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f/.tmp/info/f34c812e2f5d4c2da9e93b3441a465de is 45, key is default/info:d/1731046703700/Put/seqid=0 2024-11-08T06:21:39,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741914_1090 (size=5037) 2024-11-08T06:21:39,696 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:39,896 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 640f50ffb597620ddc4800f241b7ca8f, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:39,944 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f/.tmp/info/f34c812e2f5d4c2da9e93b3441a465de 2024-11-08T06:21:39,952 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f/.tmp/info/f34c812e2f5d4c2da9e93b3441a465de as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f/info/f34c812e2f5d4c2da9e93b3441a465de 2024-11-08T06:21:39,957 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f/info/f34c812e2f5d4c2da9e93b3441a465de, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T06:21:39,958 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 640f50ffb597620ddc4800f241b7ca8f in 435ms, sequenceid=6, compaction requested=false 2024-11-08T06:21:39,962 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/namespace/640f50ffb597620ddc4800f241b7ca8f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T06:21:39,962 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:21:39,962 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 640f50ffb597620ddc4800f241b7ca8f: 2024-11-08T06:21:39,963 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731046702323.640f50ffb597620ddc4800f241b7ca8f. 2024-11-08T06:21:39,963 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1052053f1be9a398e9e6217d04d32d31, disabling compactions & flushes 2024-11-08T06:21:39,963 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:21:39,963 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:21:39,963 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. after waiting 0 ms 2024-11-08T06:21:39,963 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:21:39,963 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1052053f1be9a398e9e6217d04d32d31 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:39,978 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31/.tmp/testFamily/96e824e167bd400887f40abe1479456a is 61, key is row-0/testFamily:testQualifier/1731046784160/Put/seqid=0 2024-11-08T06:21:39,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741915_1091 (size=5049) 2024-11-08T06:21:40,096 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 9 regions to close 2024-11-08T06:21:40,096 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1603): Online Regions={1052053f1be9a398e9e6217d04d32d31=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31., 8443cee1b4a7317c15452e64a36df3da=testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da., e60b79623b21cdaf896284976a0739d4=hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4., 0b865dba842e12e627d69ce50c784684=testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684., 79c06b38371ce93f62765ff1ce21f968=hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968., 50376f508b5217860f26c5f8a8fa984c=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c., f313d7cca28bccec72c3e8e677b94d37=hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37., 2a97b1538ceac9f3bb41ab032e3eaa06=testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06., 1bb1e1befa856ff8a272061f95c2a2d0=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.} 2024-11-08T06:21:40,096 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:40,296 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1052053f1be9a398e9e6217d04d32d31, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:40,300 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(140): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d1f4bb6aa14f:46003 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.getLiveRegionServers(RegionServerStatusProtos.java:17362) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionImplementation.getLiveRegionServers(ConnectionImplementation.java:2376) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:138) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 12 more 2024-11-08T06:21:40,383 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31/.tmp/testFamily/96e824e167bd400887f40abe1479456a 2024-11-08T06:21:40,389 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31/.tmp/testFamily/96e824e167bd400887f40abe1479456a as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31/testFamily/96e824e167bd400887f40abe1479456a 2024-11-08T06:21:40,395 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31/testFamily/96e824e167bd400887f40abe1479456a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:40,396 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1052053f1be9a398e9e6217d04d32d31 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:40,400 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/1052053f1be9a398e9e6217d04d32d31/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:40,400 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:21:40,401 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1052053f1be9a398e9e6217d04d32d31: 2024-11-08T06:21:40,401 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731046781140.1052053f1be9a398e9e6217d04d32d31. 2024-11-08T06:21:40,401 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8443cee1b4a7317c15452e64a36df3da, disabling compactions & flushes 2024-11-08T06:21:40,401 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:21:40,401 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:21:40,401 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. after waiting 0 ms 2024-11-08T06:21:40,401 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:21:40,401 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8443cee1b4a7317c15452e64a36df3da 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:40,416 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da/.tmp/testFamily/ab0e87640a78494f85295282080d7e78 is 61, key is row-0/testFamily:testQualifier/1731046821566/Put/seqid=0 2024-11-08T06:21:40,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741916_1092 (size=5049) 2024-11-08T06:21:40,497 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:40,697 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, 8443cee1b4a7317c15452e64a36df3da, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:40,820 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da/.tmp/testFamily/ab0e87640a78494f85295282080d7e78 2024-11-08T06:21:40,827 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da/.tmp/testFamily/ab0e87640a78494f85295282080d7e78 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da/testFamily/ab0e87640a78494f85295282080d7e78 2024-11-08T06:21:40,832 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da/testFamily/ab0e87640a78494f85295282080d7e78, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:40,833 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8443cee1b4a7317c15452e64a36df3da in 432ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:40,837 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/8443cee1b4a7317c15452e64a36df3da/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:40,837 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:21:40,837 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8443cee1b4a7317c15452e64a36df3da: 2024-11-08T06:21:40,837 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731046818549.8443cee1b4a7317c15452e64a36df3da. 2024-11-08T06:21:40,837 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e60b79623b21cdaf896284976a0739d4, disabling compactions & flushes 2024-11-08T06:21:40,837 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:21:40,837 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:21:40,837 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. after waiting 0 ms 2024-11-08T06:21:40,837 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:21:40,838 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e60b79623b21cdaf896284976a0739d4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:40,853 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4/.tmp/testFamily/c8aac12acb574f7ab0ef8a45ba11b529 is 61, key is row-0/testFamily:testQualifier/1731046742636/Put/seqid=0 2024-11-08T06:21:40,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741917_1093 (size=5049) 2024-11-08T06:21:40,897 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:41,097 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 7 regions to close 2024-11-08T06:21:41,097 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1603): Online Regions={e60b79623b21cdaf896284976a0739d4=hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4., 0b865dba842e12e627d69ce50c784684=testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684., 79c06b38371ce93f62765ff1ce21f968=hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968., 50376f508b5217860f26c5f8a8fa984c=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c., f313d7cca28bccec72c3e8e677b94d37=hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37., 2a97b1538ceac9f3bb41ab032e3eaa06=testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06., 1bb1e1befa856ff8a272061f95c2a2d0=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.} 2024-11-08T06:21:41,097 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, e60b79623b21cdaf896284976a0739d4, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:41,258 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4/.tmp/testFamily/c8aac12acb574f7ab0ef8a45ba11b529 2024-11-08T06:21:41,264 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4/.tmp/testFamily/c8aac12acb574f7ab0ef8a45ba11b529 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4/testFamily/c8aac12acb574f7ab0ef8a45ba11b529 2024-11-08T06:21:41,270 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4/testFamily/c8aac12acb574f7ab0ef8a45ba11b529, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:41,270 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e60b79623b21cdaf896284976a0739d4 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:41,274 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnClose_0_-false/e60b79623b21cdaf896284976a0739d4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:41,275 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:21:41,275 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e60b79623b21cdaf896284976a0739d4: 2024-11-08T06:21:41,275 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731046741587.e60b79623b21cdaf896284976a0739d4. 2024-11-08T06:21:41,275 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0b865dba842e12e627d69ce50c784684, disabling compactions & flushes 2024-11-08T06:21:41,275 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:21:41,275 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:21:41,275 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. after waiting 0 ms 2024-11-08T06:21:41,275 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:21:41,275 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0b865dba842e12e627d69ce50c784684 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:41,290 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684/.tmp/testFamily/5a30be4c742947bdb254db8540a0c973 is 61, key is row-0/testFamily:testQualifier/1731046827623/Put/seqid=0 2024-11-08T06:21:41,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741918_1094 (size=5049) 2024-11-08T06:21:41,298 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:41,498 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:41,695 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684/.tmp/testFamily/5a30be4c742947bdb254db8540a0c973 2024-11-08T06:21:41,698 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 0b865dba842e12e627d69ce50c784684, 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:41,702 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684/.tmp/testFamily/5a30be4c742947bdb254db8540a0c973 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684/testFamily/5a30be4c742947bdb254db8540a0c973 2024-11-08T06:21:41,707 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684/testFamily/5a30be4c742947bdb254db8540a0c973, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:41,708 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0b865dba842e12e627d69ce50c784684 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:41,711 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnNextAsync_1_-true/0b865dba842e12e627d69ce50c784684/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:41,712 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:21:41,712 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0b865dba842e12e627d69ce50c784684: 2024-11-08T06:21:41,712 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731046824607.0b865dba842e12e627d69ce50c784684. 2024-11-08T06:21:41,712 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 79c06b38371ce93f62765ff1ce21f968, disabling compactions & flushes 2024-11-08T06:21:41,712 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:21:41,712 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:21:41,712 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. after waiting 0 ms 2024-11-08T06:21:41,712 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:21:41,712 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 79c06b38371ce93f62765ff1ce21f968 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:41,727 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968/.tmp/testFamily/41b9600754e748978a6ac7f6280b512d is 61, key is row-0/testFamily:testQualifier/1731046775081/Put/seqid=0 2024-11-08T06:21:41,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741919_1095 (size=5049) 2024-11-08T06:21:41,898 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:42,098 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 5 regions to close 2024-11-08T06:21:42,098 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1603): Online Regions={79c06b38371ce93f62765ff1ce21f968=hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968., 50376f508b5217860f26c5f8a8fa984c=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c., f313d7cca28bccec72c3e8e677b94d37=hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37., 2a97b1538ceac9f3bb41ab032e3eaa06=testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06., 1bb1e1befa856ff8a272061f95c2a2d0=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.} 2024-11-08T06:21:42,098 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, 79c06b38371ce93f62765ff1ce21f968, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:42,132 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968/.tmp/testFamily/41b9600754e748978a6ac7f6280b512d 2024-11-08T06:21:42,139 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968/.tmp/testFamily/41b9600754e748978a6ac7f6280b512d as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968/testFamily/41b9600754e748978a6ac7f6280b512d 2024-11-08T06:21:42,145 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968/testFamily/41b9600754e748978a6ac7f6280b512d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:42,146 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 79c06b38371ce93f62765ff1ce21f968 in 434ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:42,150 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_0_-false/79c06b38371ce93f62765ff1ce21f968/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:42,150 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:21:42,150 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 79c06b38371ce93f62765ff1ce21f968: 2024-11-08T06:21:42,150 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731046772062.79c06b38371ce93f62765ff1ce21f968. 2024-11-08T06:21:42,150 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 50376f508b5217860f26c5f8a8fa984c, disabling compactions & flushes 2024-11-08T06:21:42,150 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:21:42,150 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:21:42,150 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. after waiting 0 ms 2024-11-08T06:21:42,150 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:21:42,150 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 50376f508b5217860f26c5f8a8fa984c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:42,168 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c/.tmp/testFamily/ca7113eb7e2840c49e7c44704dcac4be is 61, key is row-0/testFamily:testQualifier/1731046801365/Put/seqid=0 2024-11-08T06:21:42,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741920_1096 (size=5049) 2024-11-08T06:21:42,299 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:42,393 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T06:21:42,499 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, 50376f508b5217860f26c5f8a8fa984c, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:42,572 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c/.tmp/testFamily/ca7113eb7e2840c49e7c44704dcac4be 2024-11-08T06:21:42,579 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c/.tmp/testFamily/ca7113eb7e2840c49e7c44704dcac4be as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c/testFamily/ca7113eb7e2840c49e7c44704dcac4be 2024-11-08T06:21:42,584 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c/testFamily/ca7113eb7e2840c49e7c44704dcac4be, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:42,585 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 50376f508b5217860f26c5f8a8fa984c in 435ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:42,588 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/50376f508b5217860f26c5f8a8fa984c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:42,589 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:21:42,589 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 50376f508b5217860f26c5f8a8fa984c: 2024-11-08T06:21:42,589 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731046800349.50376f508b5217860f26c5f8a8fa984c. 2024-11-08T06:21:42,589 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f313d7cca28bccec72c3e8e677b94d37, disabling compactions & flushes 2024-11-08T06:21:42,589 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:42,589 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:42,589 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. after waiting 0 ms 2024-11-08T06:21:42,589 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:42,589 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f313d7cca28bccec72c3e8e677b94d37 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:42,604 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37/.tmp/testFamily/194394f55d194509a487a9583e8174a5 is 61, key is row-0/testFamily:testQualifier/1731046875994/Put/seqid=0 2024-11-08T06:21:42,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741921_1097 (size=5049) 2024-11-08T06:21:42,699 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:42,899 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06, f313d7cca28bccec72c3e8e677b94d37 2024-11-08T06:21:42,989 ERROR [LeaseRenewer:jenkins@localhost:45761 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:45761,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T06:21:43,009 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37/.tmp/testFamily/194394f55d194509a487a9583e8174a5 2024-11-08T06:21:43,016 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37/.tmp/testFamily/194394f55d194509a487a9583e8174a5 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37/testFamily/194394f55d194509a487a9583e8174a5 2024-11-08T06:21:43,020 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37/testFamily/194394f55d194509a487a9583e8174a5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:43,021 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f313d7cca28bccec72c3e8e677b94d37 in 432ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:43,025 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/hbase/testMetaScanTimeoutOnNext_1_-true/f313d7cca28bccec72c3e8e677b94d37/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:43,025 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:43,025 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f313d7cca28bccec72c3e8e677b94d37: 2024-11-08T06:21:43,026 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731046872979.f313d7cca28bccec72c3e8e677b94d37. 2024-11-08T06:21:43,026 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2a97b1538ceac9f3bb41ab032e3eaa06, disabling compactions & flushes 2024-11-08T06:21:43,026 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:21:43,026 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:21:43,026 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. after waiting 0 ms 2024-11-08T06:21:43,026 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:21:43,026 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2a97b1538ceac9f3bb41ab032e3eaa06 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:43,042 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06/.tmp/testFamily/c31f51ed61da464bb133637c078da0cd is 61, key is row-0/testFamily:testQualifier/1731046728230/Put/seqid=0 2024-11-08T06:21:43,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741922_1098 (size=5049) 2024-11-08T06:21:43,099 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-08T06:21:43,099 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1603): Online Regions={2a97b1538ceac9f3bb41ab032e3eaa06=testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06., 1bb1e1befa856ff8a272061f95c2a2d0=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0.} 2024-11-08T06:21:43,100 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:21:43,300 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0, 2a97b1538ceac9f3bb41ab032e3eaa06 2024-11-08T06:21:43,447 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06/.tmp/testFamily/c31f51ed61da464bb133637c078da0cd 2024-11-08T06:21:43,453 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06/.tmp/testFamily/c31f51ed61da464bb133637c078da0cd as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06/testFamily/c31f51ed61da464bb133637c078da0cd 2024-11-08T06:21:43,458 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06/testFamily/c31f51ed61da464bb133637c078da0cd, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:43,459 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2a97b1538ceac9f3bb41ab032e3eaa06 in 433ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:43,463 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testRetryOutOfOrderScannerNextException_0_-false/2a97b1538ceac9f3bb41ab032e3eaa06/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:43,463 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:21:43,463 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2a97b1538ceac9f3bb41ab032e3eaa06: 2024-11-08T06:21:43,463 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731046727211.2a97b1538ceac9f3bb41ab032e3eaa06. 2024-11-08T06:21:43,464 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1bb1e1befa856ff8a272061f95c2a2d0, disabling compactions & flushes 2024-11-08T06:21:43,464 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:43,464 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:43,464 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. after waiting 0 ms 2024-11-08T06:21:43,464 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:43,464 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1bb1e1befa856ff8a272061f95c2a2d0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T06:21:43,479 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0/.tmp/testFamily/a9ab113023e14b67b46c3215857ccb18 is 61, key is row-0/testFamily:testQualifier/1731046888051/Put/seqid=0 2024-11-08T06:21:43,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741923_1099 (size=5049) 2024-11-08T06:21:43,500 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:43,700 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1629): Waiting on 1bb1e1befa856ff8a272061f95c2a2d0 2024-11-08T06:21:43,883 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0/.tmp/testFamily/a9ab113023e14b67b46c3215857ccb18 2024-11-08T06:21:43,889 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0/.tmp/testFamily/a9ab113023e14b67b46c3215857ccb18 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0/testFamily/a9ab113023e14b67b46c3215857ccb18 2024-11-08T06:21:43,894 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0/testFamily/a9ab113023e14b67b46c3215857ccb18, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T06:21:43,895 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1bb1e1befa856ff8a272061f95c2a2d0 in 431ms, sequenceid=8, compaction requested=false 2024-11-08T06:21:43,899 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/1bb1e1befa856ff8a272061f95c2a2d0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T06:21:43,899 INFO [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:43,899 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1bb1e1befa856ff8a272061f95c2a2d0: 2024-11-08T06:21:43,900 DEBUG [RS_CLOSE_REGION-regionserver/d1f4bb6aa14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731046885036.1bb1e1befa856ff8a272061f95c2a2d0. 2024-11-08T06:21:43,900 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1250): stopping server d1f4bb6aa14f,36803,1731046638998; all regions closed. 2024-11-08T06:21:43,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741834_1010 (size=56731) 2024-11-08T06:21:43,908 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/oldWALs 2024-11-08T06:21:43,908 INFO [RS:0;d1f4bb6aa14f:36803 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL d1f4bb6aa14f%2C36803%2C1731046638998.meta:.meta(num 1731046702084) 2024-11-08T06:21:43,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741833_1009 (size=60562) 2024-11-08T06:21:43,912 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/oldWALs 2024-11-08T06:21:43,912 INFO [RS:0;d1f4bb6aa14f:36803 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL d1f4bb6aa14f%2C36803%2C1731046638998:(num 1731046701589) 2024-11-08T06:21:43,912 DEBUG [RS:0;d1f4bb6aa14f:36803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:43,912 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T06:21:43,913 INFO [RS:0;d1f4bb6aa14f:36803 {}] hbase.ChoreService(370): Chore service for: regionserver/d1f4bb6aa14f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-11-08T06:21:43,913 INFO [regionserver/d1f4bb6aa14f:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T06:21:43,914 INFO [RS:0;d1f4bb6aa14f:36803 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36803 2024-11-08T06:21:43,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d1f4bb6aa14f,36803,1731046638998 2024-11-08T06:21:43,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T06:21:43,917 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$359/0x00007f8c908d3790@7b3b54fe rejected from java.util.concurrent.ThreadPoolExecutor@3e39c5fd[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 15] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-08T06:21:43,919 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d1f4bb6aa14f,36803,1731046638998] 2024-11-08T06:21:43,919 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing d1f4bb6aa14f,36803,1731046638998; numProcessing=1 2024-11-08T06:21:43,920 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/d1f4bb6aa14f,36803,1731046638998 already deleted, retry=false 2024-11-08T06:21:43,920 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; d1f4bb6aa14f,36803,1731046638998 expired; onlineServers=0 2024-11-08T06:21:43,920 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'd1f4bb6aa14f,46003,1731046638286' ***** 2024-11-08T06:21:43,920 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T06:21:43,920 DEBUG [M:0;d1f4bb6aa14f:46003 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df947a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d1f4bb6aa14f/172.17.0.2:0 2024-11-08T06:21:43,920 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegionServer(1224): stopping server d1f4bb6aa14f,46003,1731046638286 2024-11-08T06:21:43,920 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegionServer(1250): stopping server d1f4bb6aa14f,46003,1731046638286; all regions closed. 2024-11-08T06:21:43,921 DEBUG [M:0;d1f4bb6aa14f:46003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T06:21:43,921 DEBUG [M:0;d1f4bb6aa14f:46003 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T06:21:43,921 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T06:21:43,921 DEBUG [M:0;d1f4bb6aa14f:46003 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T06:21:43,921 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster-HFileCleaner.small.0-1731046641387 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1f4bb6aa14f:0:becomeActiveMaster-HFileCleaner.small.0-1731046641387,5,FailOnTimeoutGroup] 2024-11-08T06:21:43,921 DEBUG [master/d1f4bb6aa14f:0:becomeActiveMaster-HFileCleaner.large.0-1731046641381 {}] cleaner.HFileCleaner(306): Exit Thread[master/d1f4bb6aa14f:0:becomeActiveMaster-HFileCleaner.large.0-1731046641381,5,FailOnTimeoutGroup] 2024-11-08T06:21:43,921 INFO [M:0;d1f4bb6aa14f:46003 {}] hbase.ChoreService(370): Chore service for: master/d1f4bb6aa14f:0 had [] on shutdown 2024-11-08T06:21:43,921 DEBUG [M:0;d1f4bb6aa14f:46003 {}] master.HMaster(1733): Stopping service threads 2024-11-08T06:21:43,921 INFO [M:0;d1f4bb6aa14f:46003 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T06:21:43,921 ERROR [M:0;d1f4bb6aa14f:46003 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45761,5,PEWorkerGroup] 2024-11-08T06:21:43,922 INFO [M:0;d1f4bb6aa14f:46003 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T06:21:43,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T06:21:43,922 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T06:21:43,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T06:21:43,922 DEBUG [M:0;d1f4bb6aa14f:46003 {}] zookeeper.ZKUtil(347): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T06:21:43,923 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T06:21:43,923 WARN [M:0;d1f4bb6aa14f:46003 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T06:21:43,923 INFO [M:0;d1f4bb6aa14f:46003 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-08T06:21:43,923 INFO [M:0;d1f4bb6aa14f:46003 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T06:21:43,923 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T06:21:43,923 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:21:43,923 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:21:43,923 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T06:21:43,923 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:21:43,923 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.57 KB heapSize=519.99 KB 2024-11-08T06:21:43,929 INFO [regionserver/d1f4bb6aa14f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T06:21:43,941 DEBUG [M:0;d1f4bb6aa14f:46003 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efdfbbc2b3df480499d8535c446ddf23 is 82, key is hbase:meta,,1/info:regioninfo/1731046702223/Put/seqid=0 2024-11-08T06:21:43,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741924_1100 (size=5672) 2024-11-08T06:21:44,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T06:21:44,019 INFO [RS:0;d1f4bb6aa14f:36803 {}] regionserver.HRegionServer(1307): Exiting; stopping=d1f4bb6aa14f,36803,1731046638998; zookeeper connection closed. 2024-11-08T06:21:44,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36803-0x1018686f4fc0001, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T06:21:44,019 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@395dcc77 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@395dcc77 2024-11-08T06:21:44,019 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T06:21:44,346 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efdfbbc2b3df480499d8535c446ddf23 2024-11-08T06:21:44,370 DEBUG [M:0;d1f4bb6aa14f:46003 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df09a347de834febae939ec635b10df9 is 831, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1731046704694/Put/seqid=0 2024-11-08T06:21:44,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741925_1101 (size=30294) 2024-11-08T06:21:44,775 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=448.01 KB at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df09a347de834febae939ec635b10df9 2024-11-08T06:21:44,781 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for df09a347de834febae939ec635b10df9 2024-11-08T06:21:44,797 DEBUG [M:0;d1f4bb6aa14f:46003 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc43559ff3c74804840ae479cb0bec8a is 69, key is d1f4bb6aa14f,36803,1731046638998/rs:state/1731046671426/Put/seqid=0 2024-11-08T06:21:44,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741926_1102 (size=5156) 2024-11-08T06:21:45,201 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc43559ff3c74804840ae479cb0bec8a 2024-11-08T06:21:45,207 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/efdfbbc2b3df480499d8535c446ddf23 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efdfbbc2b3df480499d8535c446ddf23 2024-11-08T06:21:45,212 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/efdfbbc2b3df480499d8535c446ddf23, entries=8, sequenceid=830, filesize=5.5 K 2024-11-08T06:21:45,213 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/df09a347de834febae939ec635b10df9 as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/df09a347de834febae939ec635b10df9 2024-11-08T06:21:45,218 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for df09a347de834febae939ec635b10df9 2024-11-08T06:21:45,218 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/df09a347de834febae939ec635b10df9, entries=92, sequenceid=830, filesize=29.6 K 2024-11-08T06:21:45,219 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc43559ff3c74804840ae479cb0bec8a as hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dc43559ff3c74804840ae479cb0bec8a 2024-11-08T06:21:45,224 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45761/user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dc43559ff3c74804840ae479cb0bec8a, entries=1, sequenceid=830, filesize=5.0 K 2024-11-08T06:21:45,225 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.57 KB/459336, heapSize ~519.70 KB/532168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1302ms, sequenceid=830, compaction requested=false 2024-11-08T06:21:45,227 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T06:21:45,227 DEBUG [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T06:21:45,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35001 is added to blk_1073741830_1006 (size=514205) 2024-11-08T06:21:45,229 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/b814aecb-5247-aca9-910f-325bf5bf118a/MasterData/WALs/d1f4bb6aa14f,46003,1731046638286/d1f4bb6aa14f%2C46003%2C1731046638286.1731046640666 not finished, retry = 0 2024-11-08T06:21:45,330 INFO [M:0;d1f4bb6aa14f:46003 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-08T06:21:45,330 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T06:21:45,330 INFO [M:0;d1f4bb6aa14f:46003 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46003 2024-11-08T06:21:45,332 DEBUG [M:0;d1f4bb6aa14f:46003 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/d1f4bb6aa14f,46003,1731046638286 already deleted, retry=false 2024-11-08T06:21:45,433 INFO [M:0;d1f4bb6aa14f:46003 {}] regionserver.HRegionServer(1307): Exiting; stopping=d1f4bb6aa14f,46003,1731046638286; zookeeper connection closed. 2024-11-08T06:21:45,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T06:21:45,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46003-0x1018686f4fc0000, quorum=127.0.0.1:49997, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T06:21:45,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@183020ad{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T06:21:45,440 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T06:21:45,440 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T06:21:45,440 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T06:21:45,441 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/hadoop.log.dir/,STOPPED} 2024-11-08T06:21:45,442 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T06:21:45,442 WARN [BP-1513049708-172.17.0.2-1731046635478 heartbeating to localhost/127.0.0.1:45761 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T06:21:45,442 WARN [BP-1513049708-172.17.0.2-1731046635478 heartbeating to localhost/127.0.0.1:45761 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1513049708-172.17.0.2-1731046635478 (Datanode Uuid a7a094ce-9504-4741-ae98-5162a1401787) service to localhost/127.0.0.1:45761 2024-11-08T06:21:45,443 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T06:21:45,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091/dfs/data/data1/current/BP-1513049708-172.17.0.2-1731046635478 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T06:21:45,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/cluster_c9005b32-44cc-d4a3-78a5-cc9986423091/dfs/data/data2/current/BP-1513049708-172.17.0.2-1731046635478 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T06:21:45,445 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T06:21:45,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T06:21:45,451 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T06:21:45,451 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T06:21:45,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T06:21:45,452 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e3c32c93-3980-0c5a-d924-91e7e0af3118/hadoop.log.dir/,STOPPED} 2024-11-08T06:21:45,460 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-08T06:21:45,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down