2024-11-07 20:17:49,844 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fd50002 2024-11-07 20:17:49,864 main DEBUG Took 0.014278 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 20:17:49,864 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 20:17:49,865 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 20:17:49,866 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 20:17:49,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,884 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 20:17:49,939 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,942 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,943 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,945 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,946 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,947 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,948 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,949 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,950 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,951 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,952 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,953 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 20:17:49,954 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,954 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 20:17:49,956 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 20:17:49,957 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 20:17:49,960 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 20:17:49,960 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 20:17:49,962 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 20:17:49,963 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 20:17:49,975 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 20:17:49,979 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 20:17:49,981 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 20:17:49,981 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 20:17:49,982 main DEBUG createAppenders(={Console}) 2024-11-07 20:17:49,983 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fd50002 initialized 2024-11-07 20:17:49,983 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fd50002 2024-11-07 20:17:49,984 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fd50002 OK. 2024-11-07 20:17:49,984 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 20:17:49,985 main DEBUG OutputStream closed 2024-11-07 20:17:49,985 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 20:17:49,985 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 20:17:49,986 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@53fdffa1 OK 2024-11-07 20:17:50,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 20:17:50,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 20:17:50,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 20:17:50,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 20:17:50,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 20:17:50,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 20:17:50,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 20:17:50,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 20:17:50,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 20:17:50,122 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 20:17:50,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 20:17:50,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 20:17:50,123 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 20:17:50,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 20:17:50,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 20:17:50,125 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 20:17:50,125 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 20:17:50,126 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 20:17:50,129 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 20:17:50,136 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5a755cc1) with optional ClassLoader: null 2024-11-07 20:17:50,137 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 20:17:50,138 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5a755cc1] started OK. 2024-11-07T20:17:50,530 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-07 20:17:50,534 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 20:17:50,535 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T20:17:50,692 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305 2024-11-07T20:17:50,775 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T20:17:50,779 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83, deleteOnExit=true 2024-11-07T20:17:50,780 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-07T20:17:50,781 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/test.cache.data in system properties and HBase conf 2024-11-07T20:17:50,782 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T20:17:50,782 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/hadoop.log.dir in system properties and HBase conf 2024-11-07T20:17:50,783 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T20:17:50,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T20:17:50,785 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-07T20:17:50,953 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T20:17:51,123 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T20:17:51,132 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T20:17:51,133 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T20:17:51,134 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T20:17:51,134 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T20:17:51,135 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T20:17:51,136 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T20:17:51,136 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T20:17:51,137 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T20:17:51,138 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T20:17:51,138 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/nfs.dump.dir in system properties and HBase conf 2024-11-07T20:17:51,139 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/java.io.tmpdir in system properties and HBase conf 2024-11-07T20:17:51,139 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T20:17:51,140 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T20:17:51,140 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T20:17:52,435 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T20:17:52,588 INFO [Time-limited test {}] log.Log(170): Logging initialized @4009ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T20:17:52,707 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T20:17:52,792 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T20:17:52,831 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T20:17:52,832 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T20:17:52,834 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T20:17:52,855 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T20:17:52,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1857c056{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/hadoop.log.dir/,AVAILABLE} 2024-11-07T20:17:52,865 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4803ff98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T20:17:53,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f422bdf{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/java.io.tmpdir/jetty-localhost-41695-hadoop-hdfs-3_4_1-tests_jar-_-any-17289640507952974472/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T20:17:53,249 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c457aa{HTTP/1.1, (http/1.1)}{localhost:41695} 2024-11-07T20:17:53,250 INFO [Time-limited test {}] server.Server(415): Started @4672ms 2024-11-07T20:17:53,933 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T20:17:53,946 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T20:17:53,952 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T20:17:53,953 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T20:17:53,953 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T20:17:53,957 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cf34ac6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/hadoop.log.dir/,AVAILABLE} 2024-11-07T20:17:53,957 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@750f55{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T20:17:54,133 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1988537a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/java.io.tmpdir/jetty-localhost-45823-hadoop-hdfs-3_4_1-tests_jar-_-any-7049836677267575044/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T20:17:54,135 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f9e40a8{HTTP/1.1, (http/1.1)}{localhost:45823} 2024-11-07T20:17:54,135 INFO [Time-limited test {}] server.Server(415): Started @5557ms 2024-11-07T20:17:54,208 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T20:17:55,019 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83/dfs/data/data1/current/BP-181548129-172.17.0.2-1731010672065/current, will proceed with Du for space computation calculation, 2024-11-07T20:17:55,020 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83/dfs/data/data2/current/BP-181548129-172.17.0.2-1731010672065/current, will proceed with Du for space computation calculation, 2024-11-07T20:17:55,126 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T20:17:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88336b77446fbf0a with lease ID 0x5a74e354c1458189: Processing first storage report for DS-808df9b5-5983-494b-bac1-a3b7ccfc6f45 from datanode DatanodeRegistration(127.0.0.1:33209, datanodeUuid=4751e2ac-fcba-46f5-8dda-83b57dedad2a, infoPort=45553, infoSecurePort=0, ipcPort=43263, storageInfo=lv=-57;cid=testClusterID;nsid=784149822;c=1731010672065) 2024-11-07T20:17:55,244 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88336b77446fbf0a with lease ID 0x5a74e354c1458189: from storage DS-808df9b5-5983-494b-bac1-a3b7ccfc6f45 node DatanodeRegistration(127.0.0.1:33209, datanodeUuid=4751e2ac-fcba-46f5-8dda-83b57dedad2a, infoPort=45553, infoSecurePort=0, ipcPort=43263, storageInfo=lv=-57;cid=testClusterID;nsid=784149822;c=1731010672065), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T20:17:55,245 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88336b77446fbf0a with lease ID 0x5a74e354c1458189: Processing first storage report for DS-d8c6981b-3d60-40ac-9146-59090c35e2d2 from datanode DatanodeRegistration(127.0.0.1:33209, datanodeUuid=4751e2ac-fcba-46f5-8dda-83b57dedad2a, infoPort=45553, infoSecurePort=0, ipcPort=43263, storageInfo=lv=-57;cid=testClusterID;nsid=784149822;c=1731010672065) 2024-11-07T20:17:55,245 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88336b77446fbf0a with lease ID 0x5a74e354c1458189: from storage DS-d8c6981b-3d60-40ac-9146-59090c35e2d2 node DatanodeRegistration(127.0.0.1:33209, datanodeUuid=4751e2ac-fcba-46f5-8dda-83b57dedad2a, infoPort=45553, infoSecurePort=0, ipcPort=43263, storageInfo=lv=-57;cid=testClusterID;nsid=784149822;c=1731010672065), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T20:17:55,285 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305 2024-11-07T20:17:55,401 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83/zookeeper_0, clientPort=59122, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T20:17:55,413 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=59122 2024-11-07T20:17:55,427 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T20:17:55,430 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T20:17:55,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741825_1001 (size=7) 2024-11-07T20:17:56,195 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 with version=8 2024-11-07T20:17:56,196 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/hbase-staging 2024-11-07T20:17:56,378 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T20:17:56,756 INFO [Time-limited test {}] client.ConnectionUtils(129): master/15c0e1002410:0 server-side Connection retries=9 2024-11-07T20:17:56,783 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T20:17:56,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T20:17:56,784 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T20:17:56,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T20:17:56,785 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T20:17:56,969 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T20:17:57,045 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T20:17:57,057 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T20:17:57,063 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T20:17:57,103 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12993 (auto-detected) 2024-11-07T20:17:57,105 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T20:17:57,150 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:46471 2024-11-07T20:17:57,161 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T20:17:57,164 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T20:17:57,180 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:46471 connecting to ZooKeeper ensemble=127.0.0.1:59122 2024-11-07T20:17:57,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464710x0, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T20:17:57,269 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46471-0x101846238170000 connected 2024-11-07T20:17:57,345 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T20:17:57,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T20:17:57,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T20:17:57,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46471 2024-11-07T20:17:57,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46471 2024-11-07T20:17:57,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46471 2024-11-07T20:17:57,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46471 2024-11-07T20:17:57,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46471 2024-11-07T20:17:57,428 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7, hbase.cluster.distributed=false 2024-11-07T20:17:57,534 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/15c0e1002410:0 server-side Connection retries=9 2024-11-07T20:17:57,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T20:17:57,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T20:17:57,535 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T20:17:57,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T20:17:57,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T20:17:57,538 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T20:17:57,541 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T20:17:57,542 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:32779 2024-11-07T20:17:57,545 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T20:17:57,560 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T20:17:57,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T20:17:57,566 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T20:17:57,571 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:32779 connecting to ZooKeeper ensemble=127.0.0.1:59122 2024-11-07T20:17:57,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:327790x0, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T20:17:57,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:327790x0, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T20:17:57,585 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:327790x0, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T20:17:57,586 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32779-0x101846238170001 connected 2024-11-07T20:17:57,587 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T20:17:57,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32779 2024-11-07T20:17:57,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32779 2024-11-07T20:17:57,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32779 2024-11-07T20:17:57,598 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32779 2024-11-07T20:17:57,598 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32779 2024-11-07T20:17:57,630 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/15c0e1002410,46471,1731010676369 2024-11-07T20:17:57,636 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;15c0e1002410:46471 2024-11-07T20:17:57,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T20:17:57,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T20:17:57,643 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/15c0e1002410,46471,1731010676369 2024-11-07T20:17:57,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T20:17:57,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T20:17:57,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:57,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:57,681 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T20:17:57,682 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T20:17:57,682 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/15c0e1002410,46471,1731010676369 from backup master directory 2024-11-07T20:17:57,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/15c0e1002410,46471,1731010676369 2024-11-07T20:17:57,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T20:17:57,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T20:17:57,689 WARN [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T20:17:57,690 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=15c0e1002410,46471,1731010676369 2024-11-07T20:17:57,693 INFO [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T20:17:57,695 INFO [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T20:17:57,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741826_1002 (size=42) 2024-11-07T20:17:58,180 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/hbase.id with ID: b8bc7563-69b8-41fd-b373-bab5ccf11c13 2024-11-07T20:17:58,264 INFO [master/15c0e1002410:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T20:17:58,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:58,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:58,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741827_1003 (size=196) 2024-11-07T20:17:58,764 INFO [master/15c0e1002410:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:17:58,767 INFO [master/15c0e1002410:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T20:17:58,791 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:17:58,797 INFO [master/15c0e1002410:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T20:17:58,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741828_1004 (size=1189) 2024-11-07T20:17:58,871 INFO [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store 2024-11-07T20:17:58,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741829_1005 (size=34) 2024-11-07T20:17:58,903 INFO [master/15c0e1002410:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T20:17:58,903 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:17:58,905 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T20:17:58,905 INFO [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:17:58,906 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:17:58,906 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T20:17:58,906 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:17:58,906 INFO [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:17:58,906 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T20:17:58,909 WARN [master/15c0e1002410:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/.initializing 2024-11-07T20:17:58,910 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/WALs/15c0e1002410,46471,1731010676369 2024-11-07T20:17:58,921 INFO [master/15c0e1002410:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T20:17:58,940 INFO [master/15c0e1002410:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15c0e1002410%2C46471%2C1731010676369, suffix=, logDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/WALs/15c0e1002410,46471,1731010676369, archiveDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/oldWALs, maxLogs=10 2024-11-07T20:17:58,975 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/WALs/15c0e1002410,46471,1731010676369/15c0e1002410%2C46471%2C1731010676369.1731010678949, exclude list is [], retry=0 2024-11-07T20:17:59,007 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33209,DS-808df9b5-5983-494b-bac1-a3b7ccfc6f45,DISK] 2024-11-07T20:17:59,011 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T20:17:59,067 INFO [master/15c0e1002410:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/WALs/15c0e1002410,46471,1731010676369/15c0e1002410%2C46471%2C1731010676369.1731010678949 2024-11-07T20:17:59,068 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45553:45553)] 2024-11-07T20:17:59,069 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:17:59,070 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:17:59,074 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,075 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T20:17:59,166 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:17:59,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T20:17:59,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T20:17:59,177 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:17:59,179 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:17:59,179 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T20:17:59,184 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:17:59,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:17:59,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T20:17:59,189 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:17:59,190 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:17:59,195 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,196 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,208 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T20:17:59,213 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T20:17:59,221 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:17:59,222 INFO [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64710476, jitterRate=-0.03573876619338989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T20:17:59,228 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T20:17:59,229 INFO [master/15c0e1002410:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T20:17:59,267 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288fb10b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:17:59,307 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-07T20:17:59,321 INFO [master/15c0e1002410:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T20:17:59,322 INFO [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T20:17:59,325 INFO [master/15c0e1002410:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T20:17:59,327 INFO [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T20:17:59,333 INFO [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 6 msec 2024-11-07T20:17:59,333 INFO [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T20:17:59,391 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T20:17:59,414 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T20:17:59,417 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-07T20:17:59,420 INFO [master/15c0e1002410:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T20:17:59,424 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T20:17:59,431 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-07T20:17:59,434 INFO [master/15c0e1002410:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T20:17:59,440 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T20:17:59,442 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-07T20:17:59,443 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T20:17:59,444 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T20:17:59,458 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T20:17:59,462 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T20:17:59,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T20:17:59,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:59,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T20:17:59,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:59,470 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=15c0e1002410,46471,1731010676369, sessionid=0x101846238170000, setting cluster-up flag (Was=false) 2024-11-07T20:17:59,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:59,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:59,517 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T20:17:59,523 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=15c0e1002410,46471,1731010676369 2024-11-07T20:17:59,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:59,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:17:59,539 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T20:17:59,541 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=15c0e1002410,46471,1731010676369 2024-11-07T20:17:59,649 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;15c0e1002410:32779 2024-11-07T20:17:59,651 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1008): ClusterId : b8bc7563-69b8-41fd-b373-bab5ccf11c13 2024-11-07T20:17:59,655 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T20:17:59,666 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T20:17:59,666 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T20:17:59,671 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T20:17:59,672 DEBUG [RS:0;15c0e1002410:32779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7565c5f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:17:59,672 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-07T20:17:59,674 DEBUG [RS:0;15c0e1002410:32779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@275590c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=15c0e1002410/172.17.0.2:0 2024-11-07T20:17:59,678 INFO [RS:0;15c0e1002410:32779 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-07T20:17:59,678 INFO [RS:0;15c0e1002410:32779 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-07T20:17:59,678 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-07T20:17:59,679 INFO [master/15c0e1002410:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-07T20:17:59,681 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3073): reportForDuty to master=15c0e1002410,46471,1731010676369 with isa=15c0e1002410/172.17.0.2:32779, startcode=1731010677532 2024-11-07T20:17:59,682 INFO [master/15c0e1002410:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T20:17:59,690 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 15c0e1002410,46471,1731010676369 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T20:17:59,695 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/15c0e1002410:0, corePoolSize=5, maxPoolSize=5 2024-11-07T20:17:59,695 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/15c0e1002410:0, corePoolSize=5, maxPoolSize=5 2024-11-07T20:17:59,695 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/15c0e1002410:0, corePoolSize=5, maxPoolSize=5 2024-11-07T20:17:59,696 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/15c0e1002410:0, corePoolSize=5, maxPoolSize=5 2024-11-07T20:17:59,696 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/15c0e1002410:0, corePoolSize=10, maxPoolSize=10 2024-11-07T20:17:59,696 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,697 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/15c0e1002410:0, corePoolSize=2, maxPoolSize=2 2024-11-07T20:17:59,697 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,698 DEBUG [RS:0;15c0e1002410:32779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T20:17:59,705 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T20:17:59,706 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-07T20:17:59,711 INFO [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731010709710 2024-11-07T20:17:59,712 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:17:59,713 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T20:17:59,712 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T20:17:59,714 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T20:17:59,718 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T20:17:59,718 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T20:17:59,720 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T20:17:59,720 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T20:17:59,723 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,724 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T20:17:59,726 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T20:17:59,726 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T20:17:59,729 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T20:17:59,732 INFO [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T20:17:59,737 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/15c0e1002410:0:becomeActiveMaster-HFileCleaner.large.0-1731010679734,5,FailOnTimeoutGroup] 2024-11-07T20:17:59,748 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/15c0e1002410:0:becomeActiveMaster-HFileCleaner.small.0-1731010679737,5,FailOnTimeoutGroup] 2024-11-07T20:17:59,748 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,749 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T20:17:59,752 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38309, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T20:17:59,751 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,753 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741831_1007 (size=1039) 2024-11-07T20:17:59,762 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46471 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 15c0e1002410,32779,1731010677532 2024-11-07T20:17:59,765 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46471 {}] master.ServerManager(486): Registering regionserver=15c0e1002410,32779,1731010677532 2024-11-07T20:17:59,785 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:17:59,785 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:39675 2024-11-07T20:17:59,785 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-07T20:17:59,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T20:17:59,792 DEBUG [RS:0;15c0e1002410:32779 {}] zookeeper.ZKUtil(111): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/15c0e1002410,32779,1731010677532 2024-11-07T20:17:59,792 WARN [RS:0;15c0e1002410:32779 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T20:17:59,792 INFO [RS:0;15c0e1002410:32779 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T20:17:59,793 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/WALs/15c0e1002410,32779,1731010677532 2024-11-07T20:17:59,794 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [15c0e1002410,32779,1731010677532] 2024-11-07T20:17:59,809 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-07T20:17:59,821 INFO [RS:0;15c0e1002410:32779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T20:17:59,835 INFO [RS:0;15c0e1002410:32779 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T20:17:59,838 INFO [RS:0;15c0e1002410:32779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T20:17:59,838 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,839 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-07T20:17:59,848 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,849 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,849 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,849 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,849 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,849 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,850 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/15c0e1002410:0, corePoolSize=2, maxPoolSize=2 2024-11-07T20:17:59,850 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,850 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,850 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,851 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,851 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/15c0e1002410:0, corePoolSize=1, maxPoolSize=1 2024-11-07T20:17:59,851 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/15c0e1002410:0, corePoolSize=3, maxPoolSize=3 2024-11-07T20:17:59,851 DEBUG [RS:0;15c0e1002410:32779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/15c0e1002410:0, corePoolSize=3, maxPoolSize=3 2024-11-07T20:17:59,852 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,852 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,853 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,853 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,853 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,32779,1731010677532-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T20:17:59,879 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T20:17:59,881 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,32779,1731010677532-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T20:17:59,910 INFO [RS:0;15c0e1002410:32779 {}] regionserver.Replication(204): 15c0e1002410,32779,1731010677532 started 2024-11-07T20:17:59,910 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1767): Serving as 15c0e1002410,32779,1731010677532, RpcServer on 15c0e1002410/172.17.0.2:32779, sessionid=0x101846238170001 2024-11-07T20:17:59,911 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T20:17:59,914 DEBUG [RS:0;15c0e1002410:32779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 15c0e1002410,32779,1731010677532 2024-11-07T20:17:59,915 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15c0e1002410,32779,1731010677532' 2024-11-07T20:17:59,915 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T20:17:59,916 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T20:17:59,917 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T20:17:59,917 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T20:17:59,917 DEBUG [RS:0;15c0e1002410:32779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 15c0e1002410,32779,1731010677532 2024-11-07T20:17:59,917 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '15c0e1002410,32779,1731010677532' 2024-11-07T20:17:59,917 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T20:17:59,918 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T20:17:59,919 DEBUG [RS:0;15c0e1002410:32779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T20:17:59,919 INFO [RS:0;15c0e1002410:32779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T20:17:59,919 INFO [RS:0;15c0e1002410:32779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T20:18:00,155 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-07T20:18:00,156 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:18:00,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741832_1008 (size=32) 2024-11-07T20:18:00,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:00,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T20:18:00,189 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T20:18:00,189 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:00,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T20:18:00,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T20:18:00,193 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T20:18:00,193 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:00,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T20:18:00,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T20:18:00,198 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T20:18:00,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:00,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T20:18:00,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740 2024-11-07T20:18:00,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740 2024-11-07T20:18:00,206 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T20:18:00,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T20:18:00,215 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:18:00,216 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69929202, jitterRate=0.04202631115913391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T20:18:00,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T20:18:00,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T20:18:00,219 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T20:18:00,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T20:18:00,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T20:18:00,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T20:18:00,220 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T20:18:00,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T20:18:00,224 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-07T20:18:00,224 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-07T20:18:00,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T20:18:00,242 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T20:18:00,245 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T20:18:00,396 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:01,397 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:02,397 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:03,398 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:04,399 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:05,399 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:06,400 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:07,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T20:18:07,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T20:18:07,401 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:08,401 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:09,402 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:10,403 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:11,404 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:12,405 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:13,406 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:14,406 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:15,407 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:16,408 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:17,409 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:18,409 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:19,410 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:20,411 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:21,412 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:22,412 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:23,413 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:24,414 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:25,280 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:18:25,414 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:25,564 ERROR [LeaseRenewer:jenkins@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39675,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:18:26,415 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:27,416 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:28,416 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:29,417 WARN [15c0e1002410:46471 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-07T20:18:29,925 INFO [RS:0;15c0e1002410:32779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T20:18:29,930 INFO [RS:0;15c0e1002410:32779 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15c0e1002410%2C32779%2C1731010677532, suffix=, logDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/WALs/15c0e1002410,32779,1731010677532, archiveDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/oldWALs, maxLogs=32 2024-11-07T20:18:29,946 DEBUG [RS:0;15c0e1002410:32779 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/WALs/15c0e1002410,32779,1731010677532/15c0e1002410%2C32779%2C1731010677532.1731010709932, exclude list is [], retry=0 2024-11-07T20:18:29,951 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33209,DS-808df9b5-5983-494b-bac1-a3b7ccfc6f45,DISK] 2024-11-07T20:18:29,954 INFO [RS:0;15c0e1002410:32779 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/WALs/15c0e1002410,32779,1731010677532/15c0e1002410%2C32779%2C1731010677532.1731010709932 2024-11-07T20:18:29,955 DEBUG [RS:0;15c0e1002410:32779 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45553:45553)] 2024-11-07T20:18:30,169 DEBUG [15c0e1002410:46471 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T20:18:30,174 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:30,179 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 15c0e1002410,32779,1731010677532, state=OPENING 2024-11-07T20:18:30,184 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T20:18:30,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:18:30,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:18:30,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T20:18:30,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T20:18:30,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:18:30,363 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:18:30,365 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T20:18:30,369 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54784, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T20:18:30,380 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-07T20:18:30,380 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T20:18:30,381 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T20:18:30,384 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=15c0e1002410%2C32779%2C1731010677532.meta, suffix=.meta, logDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/WALs/15c0e1002410,32779,1731010677532, archiveDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/oldWALs, maxLogs=32 2024-11-07T20:18:30,401 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/WALs/15c0e1002410,32779,1731010677532/15c0e1002410%2C32779%2C1731010677532.meta.1731010710386.meta, exclude list is [], retry=0 2024-11-07T20:18:30,404 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33209,DS-808df9b5-5983-494b-bac1-a3b7ccfc6f45,DISK] 2024-11-07T20:18:30,407 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/WALs/15c0e1002410,32779,1731010677532/15c0e1002410%2C32779%2C1731010677532.meta.1731010710386.meta 2024-11-07T20:18:30,408 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45553:45553)] 2024-11-07T20:18:30,408 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:18:30,409 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T20:18:30,468 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T20:18:30,473 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T20:18:30,477 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T20:18:30,477 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:30,477 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-07T20:18:30,478 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-07T20:18:30,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T20:18:30,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T20:18:30,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:30,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T20:18:30,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T20:18:30,486 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T20:18:30,486 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:30,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T20:18:30,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T20:18:30,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T20:18:30,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:30,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T20:18:30,491 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740 2024-11-07T20:18:30,493 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740 2024-11-07T20:18:30,495 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T20:18:30,498 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-07T20:18:30,500 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68896075, jitterRate=0.026631519198417664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T20:18:30,501 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-07T20:18:30,509 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731010710358 2024-11-07T20:18:30,520 DEBUG [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T20:18:30,521 INFO [RS_OPEN_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-07T20:18:30,522 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:30,524 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 15c0e1002410,32779,1731010677532, state=OPEN 2024-11-07T20:18:30,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T20:18:30,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T20:18:30,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T20:18:30,531 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T20:18:30,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T20:18:30,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=15c0e1002410,32779,1731010677532 in 342 msec 2024-11-07T20:18:30,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T20:18:30,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.3050 sec 2024-11-07T20:18:30,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 30.9540 sec 2024-11-07T20:18:30,547 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731010710547, completionTime=-1 2024-11-07T20:18:30,547 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T20:18:30,548 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-07T20:18:30,584 DEBUG [hconnection-0x3652dd57-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:30,586 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54788, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:30,596 INFO [master/15c0e1002410:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-07T20:18:30,596 INFO [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731010770596 2024-11-07T20:18:30,596 INFO [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731010830596 2024-11-07T20:18:30,596 INFO [master/15c0e1002410:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 48 msec 2024-11-07T20:18:30,616 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,46471,1731010676369-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T20:18:30,616 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,46471,1731010676369-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T20:18:30,616 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,46471,1731010676369-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T20:18:30,617 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-15c0e1002410:46471, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T20:18:30,618 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T20:18:30,622 DEBUG [master/15c0e1002410:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-07T20:18:30,625 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-07T20:18:30,627 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T20:18:30,633 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-07T20:18:30,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:18:30,636 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:30,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:18:30,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741835_1011 (size=358) 2024-11-07T20:18:31,053 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5dbcc8e35a32fe336d087243233e5f12, NAME => 'hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:18:31,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741836_1012 (size=42) 2024-11-07T20:18:31,464 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:31,464 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 5dbcc8e35a32fe336d087243233e5f12, disabling compactions & flushes 2024-11-07T20:18:31,464 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:18:31,464 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:18:31,464 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. after waiting 0 ms 2024-11-07T20:18:31,464 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:18:31,465 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:18:31,465 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5dbcc8e35a32fe336d087243233e5f12: 2024-11-07T20:18:31,467 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:18:31,473 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731010711468"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010711468"}]},"ts":"1731010711468"} 2024-11-07T20:18:31,498 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:18:31,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:18:31,503 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010711500"}]},"ts":"1731010711500"} 2024-11-07T20:18:31,507 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-07T20:18:31,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5dbcc8e35a32fe336d087243233e5f12, ASSIGN}] 2024-11-07T20:18:31,515 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=5dbcc8e35a32fe336d087243233e5f12, ASSIGN 2024-11-07T20:18:31,517 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=5dbcc8e35a32fe336d087243233e5f12, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:18:31,667 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5dbcc8e35a32fe336d087243233e5f12, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:31,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 5dbcc8e35a32fe336d087243233e5f12, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:18:31,826 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:18:31,833 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:18:31,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 5dbcc8e35a32fe336d087243233e5f12, NAME => 'hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:18:31,834 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:18:31,834 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:31,834 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:18:31,834 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:18:31,836 INFO [StoreOpener-5dbcc8e35a32fe336d087243233e5f12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:18:31,839 INFO [StoreOpener-5dbcc8e35a32fe336d087243233e5f12-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5dbcc8e35a32fe336d087243233e5f12 columnFamilyName info 2024-11-07T20:18:31,839 DEBUG [StoreOpener-5dbcc8e35a32fe336d087243233e5f12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:31,840 INFO [StoreOpener-5dbcc8e35a32fe336d087243233e5f12-1 {}] regionserver.HStore(327): Store=5dbcc8e35a32fe336d087243233e5f12/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:18:31,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:18:31,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:18:31,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:18:31,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:18:31,850 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 5dbcc8e35a32fe336d087243233e5f12; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67195462, jitterRate=0.0012904107570648193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:18:31,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 5dbcc8e35a32fe336d087243233e5f12: 2024-11-07T20:18:31,854 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12., pid=6, masterSystemTime=1731010711826 2024-11-07T20:18:31,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:18:31,857 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:18:31,858 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=5dbcc8e35a32fe336d087243233e5f12, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:31,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T20:18:31,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 5dbcc8e35a32fe336d087243233e5f12, server=15c0e1002410,32779,1731010677532 in 190 msec 2024-11-07T20:18:31,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T20:18:31,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=5dbcc8e35a32fe336d087243233e5f12, ASSIGN in 353 msec 2024-11-07T20:18:31,870 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:18:31,870 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010711870"}]},"ts":"1731010711870"} 2024-11-07T20:18:31,873 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-07T20:18:31,876 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:18:31,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2490 sec 2024-11-07T20:18:31,936 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-07T20:18:31,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-07T20:18:31,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:18:31,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:18:31,967 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-07T20:18:31,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T20:18:31,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 22 msec 2024-11-07T20:18:31,991 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-07T20:18:32,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-07T20:18:32,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec 2024-11-07T20:18:32,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-07T20:18:32,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-07T20:18:32,019 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 34.329sec 2024-11-07T20:18:32,021 INFO [master/15c0e1002410:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T20:18:32,022 INFO [master/15c0e1002410:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T20:18:32,023 INFO [master/15c0e1002410:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T20:18:32,024 INFO [master/15c0e1002410:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T20:18:32,024 INFO [master/15c0e1002410:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T20:18:32,025 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,46471,1731010676369-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T20:18:32,025 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,46471,1731010676369-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T20:18:32,032 DEBUG [master/15c0e1002410:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-07T20:18:32,033 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T20:18:32,033 INFO [master/15c0e1002410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=15c0e1002410,46471,1731010676369-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T20:18:32,072 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x551b3f6b to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e633ce7 2024-11-07T20:18:32,073 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-07T20:18:32,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5529b0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:32,084 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T20:18:32,084 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T20:18:32,095 DEBUG [hconnection-0x7939c77a-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:32,103 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54798, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:32,113 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=15c0e1002410,46471,1731010676369 2024-11-07T20:18:32,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=385, ProcessCount=11, AvailableMemoryMB=5906 2024-11-07T20:18:32,138 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5691d0bd to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@726f9398 2024-11-07T20:18:32,143 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x5691d0bd {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3da69f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:32,147 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a8b4499 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64f6f903 2024-11-07T20:18:32,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45041705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:32,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T20:18:32,162 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56702, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T20:18:32,173 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:18:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T20:18:32,177 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:18:32,177 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:32,179 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:18:32,180 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-07T20:18:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T20:18:32,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741837_1013 (size=437) 2024-11-07T20:18:32,611 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => dd525a3c56e204c238b5565ab48afa57, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:18:32,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741838_1014 (size=88) 2024-11-07T20:18:33,021 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:33,021 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing dd525a3c56e204c238b5565ab48afa57, disabling compactions & flushes 2024-11-07T20:18:33,021 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:18:33,021 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:18:33,021 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. after waiting 0 ms 2024-11-07T20:18:33,021 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:18:33,021 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:18:33,021 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for dd525a3c56e204c238b5565ab48afa57: 2024-11-07T20:18:33,023 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:18:33,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731010713023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010713023"}]},"ts":"1731010713023"} 2024-11-07T20:18:33,027 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:18:33,029 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:18:33,029 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010713029"}]},"ts":"1731010713029"} 2024-11-07T20:18:33,032 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:18:33,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dd525a3c56e204c238b5565ab48afa57, ASSIGN}] 2024-11-07T20:18:33,038 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dd525a3c56e204c238b5565ab48afa57, ASSIGN 2024-11-07T20:18:33,039 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dd525a3c56e204c238b5565ab48afa57, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:18:33,190 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=dd525a3c56e204c238b5565ab48afa57, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:33,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure dd525a3c56e204c238b5565ab48afa57, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:18:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T20:18:33,347 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:18:33,354 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:18:33,354 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => dd525a3c56e204c238b5565ab48afa57, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:18:33,355 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:18:33,355 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:33,355 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:18:33,355 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:18:33,358 INFO [StoreOpener-dd525a3c56e204c238b5565ab48afa57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:18:33,360 INFO [StoreOpener-dd525a3c56e204c238b5565ab48afa57-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd525a3c56e204c238b5565ab48afa57 columnFamilyName testFamily 2024-11-07T20:18:33,360 DEBUG [StoreOpener-dd525a3c56e204c238b5565ab48afa57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:33,361 INFO [StoreOpener-dd525a3c56e204c238b5565ab48afa57-1 {}] regionserver.HStore(327): Store=dd525a3c56e204c238b5565ab48afa57/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:18:33,363 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:18:33,363 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:18:33,366 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:18:33,370 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:18:33,370 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened dd525a3c56e204c238b5565ab48afa57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61936869, jitterRate=-0.07706873118877411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:18:33,371 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for dd525a3c56e204c238b5565ab48afa57: 2024-11-07T20:18:33,373 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57., pid=11, masterSystemTime=1731010713347 2024-11-07T20:18:33,376 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:18:33,376 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:18:33,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=dd525a3c56e204c238b5565ab48afa57, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:33,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-07T20:18:33,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure dd525a3c56e204c238b5565ab48afa57, server=15c0e1002410,32779,1731010677532 in 188 msec 2024-11-07T20:18:33,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T20:18:33,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=dd525a3c56e204c238b5565ab48afa57, ASSIGN in 348 msec 2024-11-07T20:18:33,389 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:18:33,389 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010713389"}]},"ts":"1731010713389"} 2024-11-07T20:18:33,391 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:18:33,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:18:33,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 1.2220 sec 2024-11-07T20:18:34,676 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T20:18:34,678 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56704, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T20:18:35,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-07T20:18:35,217 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-07T20:18:35,217 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:18:35,219 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:35,223 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T20:18:35,224 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:35,224 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-07T20:18:35,226 DEBUG [hconnection-0x7686ce15-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:35,228 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54808, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:35,241 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:18:35,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T20:18:35,258 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:18:35,260 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x5691d0bd {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:18:35,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:35,265 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54812, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:18:35,274 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T20:18:35,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645771 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:35,276 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T20:18:35,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645771 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:35,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905845 2024-11-07T20:18:35,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54812 deadline: 1731010718275, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:35,296 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645771 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:18:35,298 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T20:18:35,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645772 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:35,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T20:18:35,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:35,300 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:35,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645773 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:35,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905843 2024-11-07T20:18:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54812 deadline: 1731010718300, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:35,302 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645773 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:18:35,304 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:35,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645774 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:35,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6417099302534905842 2024-11-07T20:18:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54812 deadline: 1731010718303, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:35,307 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:35,306 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645774 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:18:35,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645775 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:35,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6417099302534905841 2024-11-07T20:18:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54812 deadline: 1731010718306, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:35,310 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645775 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:18:35,311 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:35,312 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a8b4499 to 127.0.0.1:59122 2024-11-07T20:18:35,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:35,313 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:18:35,313 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:18:35,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:35,318 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5691d0bd to 127.0.0.1:59122 2024-11-07T20:18:35,329 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=226 (was 219) Potentially hanging thread: hconnection-0x7939c77a-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59122@0x5691d0bd java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$732/0x00007f5f04b43410.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59122@0x6a8b4499 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$732/0x00007f5f04b43410.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59122@0x6a8b4499-SendThread(127.0.0.1:59122) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59122@0x5691d0bd-SendThread(127.0.0.1:59122) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1042884689_22 at /127.0.0.1:46412 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59122@0x5691d0bd-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RS_OPEN_REGION-regionserver/15c0e1002410:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59122@0x6a8b4499-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=456 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 385), ProcessCount=11 (was 11), AvailableMemoryMB=5868 (was 5906) 2024-11-07T20:18:35,338 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=226, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=354, ProcessCount=11, AvailableMemoryMB=5867 2024-11-07T20:18:35,339 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x033e9983 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@775ac82d 2024-11-07T20:18:35,343 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x033e9983 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fa9d42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:35,344 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d993a3a to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3add6a1c 2024-11-07T20:18:35,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286e132f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:35,350 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:18:35,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T20:18:35,353 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:18:35,354 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:35,354 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-07T20:18:35,355 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:18:35,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T20:18:35,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741839_1015 (size=423) 2024-11-07T20:18:35,767 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e9d6a79c1b635ab184225c342afd64cd, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:18:35,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741840_1016 (size=74) 2024-11-07T20:18:36,008 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:18:36,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T20:18:36,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-07T20:18:36,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-07T20:18:36,177 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:36,177 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing e9d6a79c1b635ab184225c342afd64cd, disabling compactions & flushes 2024-11-07T20:18:36,177 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:18:36,177 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:18:36,177 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. after waiting 0 ms 2024-11-07T20:18:36,177 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:18:36,177 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:18:36,177 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for e9d6a79c1b635ab184225c342afd64cd: 2024-11-07T20:18:36,179 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:18:36,180 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731010716179"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010716179"}]},"ts":"1731010716179"} 2024-11-07T20:18:36,183 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:18:36,184 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:18:36,185 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010716184"}]},"ts":"1731010716184"} 2024-11-07T20:18:36,187 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:18:36,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=e9d6a79c1b635ab184225c342afd64cd, ASSIGN}] 2024-11-07T20:18:36,193 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=e9d6a79c1b635ab184225c342afd64cd, ASSIGN 2024-11-07T20:18:36,194 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=e9d6a79c1b635ab184225c342afd64cd, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:18:36,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=e9d6a79c1b635ab184225c342afd64cd, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:36,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure e9d6a79c1b635ab184225c342afd64cd, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:18:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T20:18:36,501 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:18:36,508 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:18:36,508 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => e9d6a79c1b635ab184225c342afd64cd, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:18:36,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:18:36,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:36,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:18:36,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:18:36,511 INFO [StoreOpener-e9d6a79c1b635ab184225c342afd64cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:18:36,514 INFO [StoreOpener-e9d6a79c1b635ab184225c342afd64cd-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9d6a79c1b635ab184225c342afd64cd columnFamilyName testFamily 2024-11-07T20:18:36,514 DEBUG [StoreOpener-e9d6a79c1b635ab184225c342afd64cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:36,515 INFO [StoreOpener-e9d6a79c1b635ab184225c342afd64cd-1 {}] regionserver.HStore(327): Store=e9d6a79c1b635ab184225c342afd64cd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:18:36,516 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:18:36,517 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:18:36,520 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:18:36,523 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:18:36,524 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened e9d6a79c1b635ab184225c342afd64cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74637257, jitterRate=0.11218179762363434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:18:36,525 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for e9d6a79c1b635ab184225c342afd64cd: 2024-11-07T20:18:36,527 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd., pid=14, masterSystemTime=1731010716501 2024-11-07T20:18:36,530 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:18:36,530 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:18:36,531 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=e9d6a79c1b635ab184225c342afd64cd, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:36,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T20:18:36,537 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure e9d6a79c1b635ab184225c342afd64cd, server=15c0e1002410,32779,1731010677532 in 186 msec 2024-11-07T20:18:36,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-07T20:18:36,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=e9d6a79c1b635ab184225c342afd64cd, ASSIGN in 346 msec 2024-11-07T20:18:36,542 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:18:36,542 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010716542"}]},"ts":"1731010716542"} 2024-11-07T20:18:36,545 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:18:36,549 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:18:36,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 1.1980 sec 2024-11-07T20:18:37,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-07T20:18:37,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:18:37,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T20:18:37,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T20:18:37,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-07T20:18:37,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-07T20:18:37,044 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T20:18:37,044 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T20:18:37,045 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-07T20:18:37,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:18:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-07T20:18:38,360 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-07T20:18:38,361 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:18:38,361 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:38,365 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T20:18:38,365 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:38,365 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T20:18:38,368 DEBUG [hconnection-0x55c2245d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:38,370 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54824, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:38,382 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:18:38,386 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:18:38,387 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:18:42,011 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:18:42,053 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-07T20:18:44,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T20:18:45,410 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:18:47,389 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd., hostname=15c0e1002410,32779,1731010677532, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:18:47,390 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T20:18:47.389Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:18:47,391 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9005 2024-11-07T20:18:47,391 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d993a3a to 127.0.0.1:59122 2024-11-07T20:18:47,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:47,392 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:18:47,392 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:18:47,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:47,392 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x033e9983 to 127.0.0.1:59122 2024-11-07T20:18:47,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=222 (was 226), OpenFileDescriptor=448 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 354), ProcessCount=11 (was 11), AvailableMemoryMB=5836 (was 5867) 2024-11-07T20:18:47,415 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=5835 2024-11-07T20:18:47,420 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e238057 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@236bbfa4 2024-11-07T20:18:47,424 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6e238057 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bd96c75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:47,426 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06f5ad60 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67942cf5 2024-11-07T20:18:47,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15473968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:47,434 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:18:47,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T20:18:47,438 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:18:47,438 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:47,438 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-07T20:18:47,440 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:18:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T20:18:47,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741841_1017 (size=421) 2024-11-07T20:18:47,472 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4ae63f3930be22567e3de62358f605f4, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:18:47,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741842_1018 (size=72) 2024-11-07T20:18:47,483 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:47,483 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 4ae63f3930be22567e3de62358f605f4, disabling compactions & flushes 2024-11-07T20:18:47,483 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:18:47,483 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:18:47,483 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. after waiting 0 ms 2024-11-07T20:18:47,483 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:18:47,483 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:18:47,483 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4ae63f3930be22567e3de62358f605f4: 2024-11-07T20:18:47,485 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:18:47,486 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731010727485"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010727485"}]},"ts":"1731010727485"} 2024-11-07T20:18:47,489 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:18:47,491 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:18:47,491 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010727491"}]},"ts":"1731010727491"} 2024-11-07T20:18:47,494 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:18:47,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4ae63f3930be22567e3de62358f605f4, ASSIGN}] 2024-11-07T20:18:47,501 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4ae63f3930be22567e3de62358f605f4, ASSIGN 2024-11-07T20:18:47,505 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4ae63f3930be22567e3de62358f605f4, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:18:47,656 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=4ae63f3930be22567e3de62358f605f4, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:47,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 4ae63f3930be22567e3de62358f605f4, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:18:47,814 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:18:47,824 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:18:47,824 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 4ae63f3930be22567e3de62358f605f4, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:18:47,824 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 4ae63f3930be22567e3de62358f605f4 2024-11-07T20:18:47,824 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:47,825 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 4ae63f3930be22567e3de62358f605f4 2024-11-07T20:18:47,825 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 4ae63f3930be22567e3de62358f605f4 2024-11-07T20:18:47,827 INFO [StoreOpener-4ae63f3930be22567e3de62358f605f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4ae63f3930be22567e3de62358f605f4 2024-11-07T20:18:47,831 INFO [StoreOpener-4ae63f3930be22567e3de62358f605f4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ae63f3930be22567e3de62358f605f4 columnFamilyName testFamily 2024-11-07T20:18:47,831 DEBUG [StoreOpener-4ae63f3930be22567e3de62358f605f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:47,834 INFO [StoreOpener-4ae63f3930be22567e3de62358f605f4-1 {}] regionserver.HStore(327): Store=4ae63f3930be22567e3de62358f605f4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:18:47,835 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4 2024-11-07T20:18:47,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4 2024-11-07T20:18:47,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 4ae63f3930be22567e3de62358f605f4 2024-11-07T20:18:47,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:18:47,846 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened 4ae63f3930be22567e3de62358f605f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71713732, jitterRate=0.06861788034439087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:18:47,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 4ae63f3930be22567e3de62358f605f4: 2024-11-07T20:18:47,848 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4., pid=17, masterSystemTime=1731010727813 2024-11-07T20:18:47,852 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:18:47,852 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:18:47,853 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=4ae63f3930be22567e3de62358f605f4, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:47,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-07T20:18:47,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 4ae63f3930be22567e3de62358f605f4, server=15c0e1002410,32779,1731010677532 in 196 msec 2024-11-07T20:18:47,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T20:18:47,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=4ae63f3930be22567e3de62358f605f4, ASSIGN in 361 msec 2024-11-07T20:18:47,866 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:18:47,866 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010727866"}]},"ts":"1731010727866"} 2024-11-07T20:18:47,869 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:18:47,872 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:18:47,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 439 msec 2024-11-07T20:18:48,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-07T20:18:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-07T20:18:48,445 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-07T20:18:48,445 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:18:48,445 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:48,451 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T20:18:48,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:48,451 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T20:18:48,454 DEBUG [hconnection-0x2a0f96e5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:48,457 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48156, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:48,470 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:18:48,474 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:18:48,475 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:18:48,476 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6e238057 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:18:48,477 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6e238057 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:48,479 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48160, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:18:48,483 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:18:48,483 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645787 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:48,485 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:18:48,485 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645787 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:48,486 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T20:18:51,911 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:18:53,354 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:18:55,281 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:18:57,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T20:18:57,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:18:57,489 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9004 2024-11-07T20:18:57,489 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x06f5ad60 to 127.0.0.1:59122 2024-11-07T20:18:57,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:57,490 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:18:57,489 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645787 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:18:57,490 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:18:57,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:57,490 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e238057 to 127.0.0.1:59122 2024-11-07T20:18:57,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905829' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T20:18:57,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:48160 deadline: 1731010743489, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905829' does not exist 2024-11-07T20:18:57,494 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:18:57,490 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 15c0e1002410,32779,1731010677532 for closing scanner id = -6417099302534905829 for 4ae63f3930be22567e3de62358f605f4 of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T20:18:57,505 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=221 (was 222), OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=378 (was 339) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5782 (was 5835) 2024-11-07T20:18:57,529 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=221, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=378, ProcessCount=11, AvailableMemoryMB=5781 2024-11-07T20:18:57,531 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x463e8dc8 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3124aff5 2024-11-07T20:18:57,554 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x463e8dc8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f08cb28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:57,556 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d1d20b1 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3171b8a3 2024-11-07T20:18:57,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3932388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:57,620 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:18:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T20:18:57,624 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:18:57,625 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:57,625 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-07T20:18:57,626 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:18:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T20:18:57,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741843_1019 (size=432) 2024-11-07T20:18:57,728 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 61a760d6216fe8653467451e266dc60e, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:18:57,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741844_1020 (size=83) 2024-11-07T20:18:57,774 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:57,774 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 61a760d6216fe8653467451e266dc60e, disabling compactions & flushes 2024-11-07T20:18:57,775 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:18:57,775 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:18:57,775 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. after waiting 0 ms 2024-11-07T20:18:57,775 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:18:57,775 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:18:57,775 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 61a760d6216fe8653467451e266dc60e: 2024-11-07T20:18:57,777 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:18:57,778 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731010737777"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010737777"}]},"ts":"1731010737777"} 2024-11-07T20:18:57,781 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:18:57,783 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:18:57,784 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010737783"}]},"ts":"1731010737783"} 2024-11-07T20:18:57,788 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:18:57,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=61a760d6216fe8653467451e266dc60e, ASSIGN}] 2024-11-07T20:18:57,798 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=61a760d6216fe8653467451e266dc60e, ASSIGN 2024-11-07T20:18:57,804 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=61a760d6216fe8653467451e266dc60e, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:18:57,955 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=61a760d6216fe8653467451e266dc60e, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:57,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure 61a760d6216fe8653467451e266dc60e, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:18:57,987 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:18:58,115 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:18:58,143 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:18:58,143 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => 61a760d6216fe8653467451e266dc60e, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:18:58,144 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false 61a760d6216fe8653467451e266dc60e 2024-11-07T20:18:58,144 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:58,144 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for 61a760d6216fe8653467451e266dc60e 2024-11-07T20:18:58,144 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for 61a760d6216fe8653467451e266dc60e 2024-11-07T20:18:58,148 INFO [StoreOpener-61a760d6216fe8653467451e266dc60e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 61a760d6216fe8653467451e266dc60e 2024-11-07T20:18:58,153 INFO [StoreOpener-61a760d6216fe8653467451e266dc60e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61a760d6216fe8653467451e266dc60e columnFamilyName testFamily 2024-11-07T20:18:58,153 DEBUG [StoreOpener-61a760d6216fe8653467451e266dc60e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:58,154 INFO [StoreOpener-61a760d6216fe8653467451e266dc60e-1 {}] regionserver.HStore(327): Store=61a760d6216fe8653467451e266dc60e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:18:58,156 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e 2024-11-07T20:18:58,158 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e 2024-11-07T20:18:58,164 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for 61a760d6216fe8653467451e266dc60e 2024-11-07T20:18:58,173 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:18:58,174 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened 61a760d6216fe8653467451e266dc60e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71183547, jitterRate=0.06071750819683075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:18:58,175 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for 61a760d6216fe8653467451e266dc60e: 2024-11-07T20:18:58,180 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e., pid=20, masterSystemTime=1731010738114 2024-11-07T20:18:58,184 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:18:58,184 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:18:58,185 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=61a760d6216fe8653467451e266dc60e, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:58,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T20:18:58,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure 61a760d6216fe8653467451e266dc60e, server=15c0e1002410,32779,1731010677532 in 230 msec 2024-11-07T20:18:58,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-07T20:18:58,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=61a760d6216fe8653467451e266dc60e, ASSIGN in 397 msec 2024-11-07T20:18:58,198 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:18:58,199 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010738198"}]},"ts":"1731010738198"} 2024-11-07T20:18:58,202 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:18:58,206 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:18:58,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 587 msec 2024-11-07T20:18:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-07T20:18:58,634 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-07T20:18:58,634 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:18:58,635 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:58,648 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-07T20:18:58,648 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:58,648 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-07T20:18:58,659 DEBUG [hconnection-0x31fd3111-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:58,671 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32792, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:58,693 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:18:58,700 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T20:18:58,704 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T20:18:58,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645791 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:58,709 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T20:18:58,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645791 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:58,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905825 2024-11-07T20:18:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:32792 deadline: 1731010739714, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:58,717 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:18:58,721 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T20:18:58,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645792 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:58,730 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T20:18:58,730 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:58,732 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:58,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645793 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:58,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905823 2024-11-07T20:18:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:32792 deadline: 1731010739733, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:58,735 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:18:58,737 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:58,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645794 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:58,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6417099302534905822 2024-11-07T20:18:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:32792 deadline: 1731010739737, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:58,739 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:18:58,741 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:58,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645795 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:18:58,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6417099302534905821 2024-11-07T20:18:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:32792 deadline: 1731010739742, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:18:58,743 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:18:58,744 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:18:58,745 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d1d20b1 to 127.0.0.1:59122 2024-11-07T20:18:58,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:58,745 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:18:58,745 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:18:58,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:18:58,746 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x463e8dc8 to 127.0.0.1:59122 2024-11-07T20:18:58,760 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=226 (was 221) - Thread LEAK? -, OpenFileDescriptor=454 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 378) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5740 (was 5781) 2024-11-07T20:18:58,773 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=5739 2024-11-07T20:18:58,775 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f750215 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@685a0b63 2024-11-07T20:18:58,779 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6f750215 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66ae37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:58,781 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7680e15b to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4486b179 2024-11-07T20:18:58,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a46656, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:18:58,791 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:18:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T20:18:58,796 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:18:58,796 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:58,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-07T20:18:58,798 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:18:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T20:18:58,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741845_1021 (size=426) 2024-11-07T20:18:58,828 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 5506bb10abae8b4dfe00662cb6db8777, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:18:58,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741846_1022 (size=77) 2024-11-07T20:18:59,243 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:59,243 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 5506bb10abae8b4dfe00662cb6db8777, disabling compactions & flushes 2024-11-07T20:18:59,243 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:18:59,243 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:18:59,243 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. after waiting 0 ms 2024-11-07T20:18:59,243 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:18:59,243 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:18:59,244 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 5506bb10abae8b4dfe00662cb6db8777: 2024-11-07T20:18:59,246 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:18:59,246 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731010739246"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010739246"}]},"ts":"1731010739246"} 2024-11-07T20:18:59,249 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:18:59,251 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:18:59,251 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010739251"}]},"ts":"1731010739251"} 2024-11-07T20:18:59,254 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:18:59,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=5506bb10abae8b4dfe00662cb6db8777, ASSIGN}] 2024-11-07T20:18:59,266 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=5506bb10abae8b4dfe00662cb6db8777, ASSIGN 2024-11-07T20:18:59,268 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=5506bb10abae8b4dfe00662cb6db8777, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:18:59,419 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=5506bb10abae8b4dfe00662cb6db8777, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:59,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 5506bb10abae8b4dfe00662cb6db8777, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:18:59,575 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:18:59,583 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:18:59,583 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => 5506bb10abae8b4dfe00662cb6db8777, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:18:59,584 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:18:59,584 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:18:59,585 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for 5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:18:59,585 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for 5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:18:59,591 INFO [StoreOpener-5506bb10abae8b4dfe00662cb6db8777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:18:59,595 INFO [StoreOpener-5506bb10abae8b4dfe00662cb6db8777-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5506bb10abae8b4dfe00662cb6db8777 columnFamilyName testFamily 2024-11-07T20:18:59,595 DEBUG [StoreOpener-5506bb10abae8b4dfe00662cb6db8777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:18:59,597 INFO [StoreOpener-5506bb10abae8b4dfe00662cb6db8777-1 {}] regionserver.HStore(327): Store=5506bb10abae8b4dfe00662cb6db8777/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:18:59,600 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:18:59,601 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:18:59,608 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for 5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:18:59,615 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:18:59,616 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened 5506bb10abae8b4dfe00662cb6db8777; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71327040, jitterRate=0.06285572052001953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:18:59,617 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for 5506bb10abae8b4dfe00662cb6db8777: 2024-11-07T20:18:59,619 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777., pid=23, masterSystemTime=1731010739575 2024-11-07T20:18:59,623 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:18:59,623 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:18:59,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=5506bb10abae8b4dfe00662cb6db8777, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:18:59,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-07T20:18:59,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 5506bb10abae8b4dfe00662cb6db8777, server=15c0e1002410,32779,1731010677532 in 206 msec 2024-11-07T20:18:59,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T20:18:59,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=5506bb10abae8b4dfe00662cb6db8777, ASSIGN in 368 msec 2024-11-07T20:18:59,644 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:18:59,648 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010739644"}]},"ts":"1731010739644"} 2024-11-07T20:18:59,651 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:18:59,655 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:18:59,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 863 msec 2024-11-07T20:18:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-07T20:18:59,802 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-07T20:18:59,802 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:18:59,802 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:59,809 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T20:18:59,809 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:18:59,809 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T20:18:59,821 DEBUG [hconnection-0x1b2cfaec-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:18:59,826 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50180, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:18:59,855 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:18:59,862 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:18:59,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:18:59,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T20:18:59,951 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39675,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:19:00,057 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-07T20:19:00,058 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-07T20:19:00,652 ERROR [LeaseRenewer:jenkins@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39675,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:19:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T20:19:01,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:19:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:50180 deadline: 1731010742873, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:19:01,890 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:19:01,891 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645800 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:19:01,892 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2029 2024-11-07T20:19:01,892 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7680e15b to 127.0.0.1:59122 2024-11-07T20:19:01,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:01,892 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:01,893 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:01,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:01,893 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f750215 to 127.0.0.1:59122 2024-11-07T20:19:01,902 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=225 (was 226), OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=5755 (was 5739) - AvailableMemoryMB LEAK? - 2024-11-07T20:19:01,911 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=5755 2024-11-07T20:19:01,913 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a8f66ab to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@715c166f 2024-11-07T20:19:01,916 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x4a8f66ab {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd7703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:01,918 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61fd6cd4 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7cbeb9fb 2024-11-07T20:19:01,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565ae6df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:01,928 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T20:19:01,931 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:01,931 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:01,931 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-07T20:19:01,932 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T20:19:01,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741847_1023 (size=427) 2024-11-07T20:19:02,344 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => aa9aeb512dbdd6d51b6c45aacf3be68c, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:02,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741848_1024 (size=78) 2024-11-07T20:19:02,754 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:02,755 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing aa9aeb512dbdd6d51b6c45aacf3be68c, disabling compactions & flushes 2024-11-07T20:19:02,755 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:19:02,755 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:19:02,755 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. after waiting 0 ms 2024-11-07T20:19:02,755 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:19:02,755 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:19:02,755 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for aa9aeb512dbdd6d51b6c45aacf3be68c: 2024-11-07T20:19:02,757 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:02,757 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731010742757"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010742757"}]},"ts":"1731010742757"} 2024-11-07T20:19:02,759 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:02,760 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:02,760 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010742760"}]},"ts":"1731010742760"} 2024-11-07T20:19:02,762 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:02,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=aa9aeb512dbdd6d51b6c45aacf3be68c, ASSIGN}] 2024-11-07T20:19:02,767 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=aa9aeb512dbdd6d51b6c45aacf3be68c, ASSIGN 2024-11-07T20:19:02,768 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=aa9aeb512dbdd6d51b6c45aacf3be68c, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:02,919 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=aa9aeb512dbdd6d51b6c45aacf3be68c, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:02,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure aa9aeb512dbdd6d51b6c45aacf3be68c, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T20:19:03,075 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:03,080 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:19:03,080 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => aa9aeb512dbdd6d51b6c45aacf3be68c, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:03,081 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:19:03,081 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:03,081 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:19:03,081 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:19:03,083 INFO [StoreOpener-aa9aeb512dbdd6d51b6c45aacf3be68c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:19:03,085 INFO [StoreOpener-aa9aeb512dbdd6d51b6c45aacf3be68c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9aeb512dbdd6d51b6c45aacf3be68c columnFamilyName testFamily 2024-11-07T20:19:03,085 DEBUG [StoreOpener-aa9aeb512dbdd6d51b6c45aacf3be68c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:03,086 INFO [StoreOpener-aa9aeb512dbdd6d51b6c45aacf3be68c-1 {}] regionserver.HStore(327): Store=aa9aeb512dbdd6d51b6c45aacf3be68c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:03,087 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:19:03,087 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:19:03,090 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:19:03,093 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:03,093 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened aa9aeb512dbdd6d51b6c45aacf3be68c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71862925, jitterRate=0.07084102928638458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:03,094 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for aa9aeb512dbdd6d51b6c45aacf3be68c: 2024-11-07T20:19:03,095 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., pid=26, masterSystemTime=1731010743075 2024-11-07T20:19:03,097 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:19:03,097 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:19:03,098 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=aa9aeb512dbdd6d51b6c45aacf3be68c, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:03,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T20:19:03,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure aa9aeb512dbdd6d51b6c45aacf3be68c, server=15c0e1002410,32779,1731010677532 in 178 msec 2024-11-07T20:19:03,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-07T20:19:03,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=aa9aeb512dbdd6d51b6c45aacf3be68c, ASSIGN in 336 msec 2024-11-07T20:19:03,106 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:03,106 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010743106"}]},"ts":"1731010743106"} 2024-11-07T20:19:03,108 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:03,111 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:03,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 1.1830 sec 2024-11-07T20:19:03,646 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:19:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-07T20:19:04,937 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-07T20:19:04,937 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:04,937 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:04,942 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:04,942 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:04,942 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-07T20:19:04,945 DEBUG [hconnection-0x3b919e1f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:04,947 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50182, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:04,966 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:04,970 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:19:04,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:19:06,059 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-07T20:19:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T20:19:06,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:19:07,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T20:19:07,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:07,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-07T20:19:07,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:07,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T20:19:07,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:07,978 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., hostname=15c0e1002410,32779,1731010677532, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:19:07,979 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T20:19:07.979Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5020: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=986ms, rpcTimeout=979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:19:07,979 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3009 2024-11-07T20:19:07,980 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61fd6cd4 to 127.0.0.1:59122 2024-11-07T20:19:07,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:07,980 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:07,980 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:07,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:07,980 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a8f66ab to 127.0.0.1:59122 2024-11-07T20:19:07,989 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=225 (was 225), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=5751 (was 5755) 2024-11-07T20:19:07,998 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=5752 2024-11-07T20:19:08,000 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x479d10ee to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ec29193 2024-11-07T20:19:08,003 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x479d10ee {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c022111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:08,005 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6aa9d0ab to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@95ecbea 2024-11-07T20:19:08,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266bb04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:08,011 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T20:19:08,013 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:08,013 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:08,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-07T20:19:08,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T20:19:08,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741849_1025 (size=425) 2024-11-07T20:19:08,426 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d1553686ddc24d1772f86227f55ddbce, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:08,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741850_1026 (size=76) 2024-11-07T20:19:08,439 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:08,439 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing d1553686ddc24d1772f86227f55ddbce, disabling compactions & flushes 2024-11-07T20:19:08,440 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:19:08,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:19:08,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. after waiting 0 ms 2024-11-07T20:19:08,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:19:08,440 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:19:08,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for d1553686ddc24d1772f86227f55ddbce: 2024-11-07T20:19:08,441 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:08,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731010748441"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010748441"}]},"ts":"1731010748441"} 2024-11-07T20:19:08,444 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:08,445 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:08,445 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010748445"}]},"ts":"1731010748445"} 2024-11-07T20:19:08,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:08,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d1553686ddc24d1772f86227f55ddbce, ASSIGN}] 2024-11-07T20:19:08,453 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d1553686ddc24d1772f86227f55ddbce, ASSIGN 2024-11-07T20:19:08,457 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d1553686ddc24d1772f86227f55ddbce, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:08,493 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:19:08,583 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:19:08,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=d1553686ddc24d1772f86227f55ddbce, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:08,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure d1553686ddc24d1772f86227f55ddbce, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:08,764 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:08,784 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:19:08,784 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => d1553686ddc24d1772f86227f55ddbce, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:08,785 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:19:08,785 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:08,785 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:19:08,785 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:19:08,793 INFO [StoreOpener-d1553686ddc24d1772f86227f55ddbce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:19:08,795 INFO [StoreOpener-d1553686ddc24d1772f86227f55ddbce-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1553686ddc24d1772f86227f55ddbce columnFamilyName testFamily 2024-11-07T20:19:08,796 DEBUG [StoreOpener-d1553686ddc24d1772f86227f55ddbce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:08,797 INFO [StoreOpener-d1553686ddc24d1772f86227f55ddbce-1 {}] regionserver.HStore(327): Store=d1553686ddc24d1772f86227f55ddbce/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:08,798 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:19:08,799 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:19:08,803 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:19:08,807 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:08,809 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened d1553686ddc24d1772f86227f55ddbce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61283016, jitterRate=-0.08681190013885498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:08,810 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for d1553686ddc24d1772f86227f55ddbce: 2024-11-07T20:19:08,811 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce., pid=29, masterSystemTime=1731010748764 2024-11-07T20:19:08,816 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:19:08,816 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:19:08,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=d1553686ddc24d1772f86227f55ddbce, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:08,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-07T20:19:08,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure d1553686ddc24d1772f86227f55ddbce, server=15c0e1002410,32779,1731010677532 in 208 msec 2024-11-07T20:19:08,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T20:19:08,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=d1553686ddc24d1772f86227f55ddbce, ASSIGN in 372 msec 2024-11-07T20:19:08,827 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:08,827 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010748827"}]},"ts":"1731010748827"} 2024-11-07T20:19:08,830 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:08,840 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:08,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 829 msec 2024-11-07T20:19:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-07T20:19:09,019 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-07T20:19:09,019 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:09,020 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:09,026 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:09,026 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:09,026 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-07T20:19:09,030 DEBUG [hconnection-0xccbe6a7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:09,033 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50188, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:09,055 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:09,064 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:19:09,065 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:19:09,068 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x479d10ee {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:19:09,068 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x479d10ee {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:09,071 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50200, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:09,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:19:09,078 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:19:09,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645812 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:19:09,080 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:19:09,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645812 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:19:09,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T20:19:12,061 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-07T20:19:12,089 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3008 2024-11-07T20:19:12,089 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6aa9d0ab to 127.0.0.1:59122 2024-11-07T20:19:12,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:12,089 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:12,089 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645812 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:19:12,090 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:12,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:12,090 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x479d10ee to 127.0.0.1:59122 2024-11-07T20:19:12,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905804' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T20:19:12,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:50200 deadline: 1731010753089, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905804' does not exist 2024-11-07T20:19:12,090 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:19:12,090 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 15c0e1002410,32779,1731010677532 for closing scanner id = -6417099302534905804 for d1553686ddc24d1772f86227f55ddbce of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T20:19:12,102 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=228 (was 225) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=5722 (was 5752) 2024-11-07T20:19:12,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=5722 2024-11-07T20:19:12,115 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a1e7cdf to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ef86722 2024-11-07T20:19:12,119 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x3a1e7cdf {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2235ec52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:12,121 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x02b1e2c0 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c7be1eb 2024-11-07T20:19:12,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ca8a3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:12,129 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:12,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-07T20:19:12,132 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:12,132 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:12,133 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-07T20:19:12,134 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T20:19:12,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741851_1027 (size=417) 2024-11-07T20:19:12,151 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3a0e4a19c977ab180d4a99c8159e7bfa, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:12,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741852_1028 (size=68) 2024-11-07T20:19:12,163 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:12,163 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 3a0e4a19c977ab180d4a99c8159e7bfa, disabling compactions & flushes 2024-11-07T20:19:12,163 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:19:12,163 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:19:12,163 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. after waiting 0 ms 2024-11-07T20:19:12,163 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:19:12,163 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:19:12,163 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3a0e4a19c977ab180d4a99c8159e7bfa: 2024-11-07T20:19:12,165 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:12,165 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1731010752165"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010752165"}]},"ts":"1731010752165"} 2024-11-07T20:19:12,168 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:12,169 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:12,170 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010752169"}]},"ts":"1731010752169"} 2024-11-07T20:19:12,172 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:12,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3a0e4a19c977ab180d4a99c8159e7bfa, ASSIGN}] 2024-11-07T20:19:12,178 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3a0e4a19c977ab180d4a99c8159e7bfa, ASSIGN 2024-11-07T20:19:12,179 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3a0e4a19c977ab180d4a99c8159e7bfa, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:12,330 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=3a0e4a19c977ab180d4a99c8159e7bfa, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:12,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 3a0e4a19c977ab180d4a99c8159e7bfa, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:12,487 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:12,494 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:19:12,494 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => 3a0e4a19c977ab180d4a99c8159e7bfa, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:12,495 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:19:12,495 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:12,495 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for 3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:19:12,495 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for 3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:19:12,497 INFO [StoreOpener-3a0e4a19c977ab180d4a99c8159e7bfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:19:12,499 INFO [StoreOpener-3a0e4a19c977ab180d4a99c8159e7bfa-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a0e4a19c977ab180d4a99c8159e7bfa columnFamilyName testFamily 2024-11-07T20:19:12,500 DEBUG [StoreOpener-3a0e4a19c977ab180d4a99c8159e7bfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:12,500 INFO [StoreOpener-3a0e4a19c977ab180d4a99c8159e7bfa-1 {}] regionserver.HStore(327): Store=3a0e4a19c977ab180d4a99c8159e7bfa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:12,502 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:19:12,502 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:19:12,507 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for 3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:19:12,511 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:12,512 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened 3a0e4a19c977ab180d4a99c8159e7bfa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62757787, jitterRate=-0.06483609974384308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:12,513 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for 3a0e4a19c977ab180d4a99c8159e7bfa: 2024-11-07T20:19:12,514 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa., pid=32, masterSystemTime=1731010752487 2024-11-07T20:19:12,517 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:19:12,517 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:19:12,518 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=3a0e4a19c977ab180d4a99c8159e7bfa, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:12,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T20:19:12,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 3a0e4a19c977ab180d4a99c8159e7bfa, server=15c0e1002410,32779,1731010677532 in 187 msec 2024-11-07T20:19:12,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T20:19:12,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=3a0e4a19c977ab180d4a99c8159e7bfa, ASSIGN in 348 msec 2024-11-07T20:19:12,529 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:12,529 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010752529"}]},"ts":"1731010752529"} 2024-11-07T20:19:12,531 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:12,535 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:12,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 406 msec 2024-11-07T20:19:12,582 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:19:12,627 INFO [master/15c0e1002410:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T20:19:12,627 INFO [master/15c0e1002410:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T20:19:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-07T20:19:13,137 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-07T20:19:13,138 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:13,138 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:13,144 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:13,144 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:13,144 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-07T20:19:13,147 DEBUG [hconnection-0x47715e82-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:13,149 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58324, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:13,165 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:13,170 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:19:13,175 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645816 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:19:13,176 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T20:19:14,287 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:19:17,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-07T20:19:17,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:17,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-07T20:19:17,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:18,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-07T20:19:19,676 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6506 2024-11-07T20:19:19,677 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x02b1e2c0 to 127.0.0.1:59122 2024-11-07T20:19:19,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:19,677 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:19,677 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:19,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:19,677 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a1e7cdf to 127.0.0.1:59122 2024-11-07T20:19:19,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 228), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=379 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=5698 (was 5722) 2024-11-07T20:19:19,698 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=379, ProcessCount=11, AvailableMemoryMB=5698 2024-11-07T20:19:19,701 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cd84775 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@246233e2 2024-11-07T20:19:19,705 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x0cd84775 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358eb924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:19,707 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x292ea26f to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5719ff4 2024-11-07T20:19:19,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b5c83c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:19,715 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-07T20:19:19,718 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:19,718 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:19,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-07T20:19:19,720 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T20:19:19,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741853_1029 (size=421) 2024-11-07T20:19:19,737 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 45f0f6007cab85b2d5717f89f96bb49b, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:19,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741854_1030 (size=72) 2024-11-07T20:19:19,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:19,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 45f0f6007cab85b2d5717f89f96bb49b, disabling compactions & flushes 2024-11-07T20:19:19,753 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:19:19,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:19:19,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. after waiting 0 ms 2024-11-07T20:19:19,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:19:19,753 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:19:19,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 45f0f6007cab85b2d5717f89f96bb49b: 2024-11-07T20:19:19,755 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:19,756 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731010759755"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010759755"}]},"ts":"1731010759755"} 2024-11-07T20:19:19,758 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:19,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:19,760 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010759760"}]},"ts":"1731010759760"} 2024-11-07T20:19:19,762 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:19,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=45f0f6007cab85b2d5717f89f96bb49b, ASSIGN}] 2024-11-07T20:19:19,769 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=45f0f6007cab85b2d5717f89f96bb49b, ASSIGN 2024-11-07T20:19:19,770 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=45f0f6007cab85b2d5717f89f96bb49b, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:19,921 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=45f0f6007cab85b2d5717f89f96bb49b, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:19,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure 45f0f6007cab85b2d5717f89f96bb49b, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:20,077 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:20,083 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:19:20,084 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => 45f0f6007cab85b2d5717f89f96bb49b, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:20,084 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false 45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:19:20,085 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:20,085 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for 45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:19:20,085 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for 45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:19:20,087 INFO [StoreOpener-45f0f6007cab85b2d5717f89f96bb49b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:19:20,089 INFO [StoreOpener-45f0f6007cab85b2d5717f89f96bb49b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45f0f6007cab85b2d5717f89f96bb49b columnFamilyName testFamily 2024-11-07T20:19:20,089 DEBUG [StoreOpener-45f0f6007cab85b2d5717f89f96bb49b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:20,090 INFO [StoreOpener-45f0f6007cab85b2d5717f89f96bb49b-1 {}] regionserver.HStore(327): Store=45f0f6007cab85b2d5717f89f96bb49b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:20,091 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:19:20,092 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:19:20,096 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for 45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:19:20,100 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:20,101 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened 45f0f6007cab85b2d5717f89f96bb49b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60880350, jitterRate=-0.09281209111213684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:20,102 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for 45f0f6007cab85b2d5717f89f96bb49b: 2024-11-07T20:19:20,103 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b., pid=35, masterSystemTime=1731010760077 2024-11-07T20:19:20,106 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:19:20,107 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:19:20,108 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=45f0f6007cab85b2d5717f89f96bb49b, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:20,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-07T20:19:20,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure 45f0f6007cab85b2d5717f89f96bb49b, server=15c0e1002410,32779,1731010677532 in 187 msec 2024-11-07T20:19:20,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-07T20:19:20,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=45f0f6007cab85b2d5717f89f96bb49b, ASSIGN in 347 msec 2024-11-07T20:19:20,117 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:20,118 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010760117"}]},"ts":"1731010760117"} 2024-11-07T20:19:20,119 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:20,123 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:20,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 408 msec 2024-11-07T20:19:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-07T20:19:20,726 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-07T20:19:20,726 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:20,727 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:20,732 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:20,732 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:20,732 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-07T20:19:20,735 DEBUG [hconnection-0x41fdda5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:20,737 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33648, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:20,755 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:20,767 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:19:20,774 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645820 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:19:20,774 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T20:19:22,275 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1508 2024-11-07T20:19:22,276 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x292ea26f to 127.0.0.1:59122 2024-11-07T20:19:22,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:22,276 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:22,276 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:22,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:22,277 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0cd84775 to 127.0.0.1:59122 2024-11-07T20:19:22,289 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=454 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=379 (was 379), ProcessCount=11 (was 11), AvailableMemoryMB=5681 (was 5698) 2024-11-07T20:19:22,299 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=379, ProcessCount=11, AvailableMemoryMB=5681 2024-11-07T20:19:22,302 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29caf2bf to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58063b0a 2024-11-07T20:19:22,305 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x29caf2bf {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2843aea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:22,307 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22f4f96b to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c6ff954 2024-11-07T20:19:22,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624eaf6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:22,314 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-07T20:19:22,317 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:22,317 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:22,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-07T20:19:22,318 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T20:19:22,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741855_1031 (size=420) 2024-11-07T20:19:22,734 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2f3f6d4a4f2ed609a76a060c63de295e, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:22,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741856_1032 (size=71) 2024-11-07T20:19:22,744 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:22,744 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2f3f6d4a4f2ed609a76a060c63de295e, disabling compactions & flushes 2024-11-07T20:19:22,744 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:19:22,744 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:19:22,744 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. after waiting 0 ms 2024-11-07T20:19:22,744 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:19:22,744 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:19:22,745 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2f3f6d4a4f2ed609a76a060c63de295e: 2024-11-07T20:19:22,746 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:22,746 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731010762746"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010762746"}]},"ts":"1731010762746"} 2024-11-07T20:19:22,750 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:22,751 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:22,751 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010762751"}]},"ts":"1731010762751"} 2024-11-07T20:19:22,753 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:22,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2f3f6d4a4f2ed609a76a060c63de295e, ASSIGN}] 2024-11-07T20:19:22,761 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2f3f6d4a4f2ed609a76a060c63de295e, ASSIGN 2024-11-07T20:19:22,762 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2f3f6d4a4f2ed609a76a060c63de295e, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:22,913 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=2f3f6d4a4f2ed609a76a060c63de295e, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:22,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 2f3f6d4a4f2ed609a76a060c63de295e, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:23,069 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:23,075 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:19:23,076 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 2f3f6d4a4f2ed609a76a060c63de295e, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:23,076 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:19:23,076 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:23,076 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:19:23,076 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:19:23,080 INFO [StoreOpener-2f3f6d4a4f2ed609a76a060c63de295e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:19:23,085 INFO [StoreOpener-2f3f6d4a4f2ed609a76a060c63de295e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f3f6d4a4f2ed609a76a060c63de295e columnFamilyName testFamily 2024-11-07T20:19:23,085 DEBUG [StoreOpener-2f3f6d4a4f2ed609a76a060c63de295e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:23,087 INFO [StoreOpener-2f3f6d4a4f2ed609a76a060c63de295e-1 {}] regionserver.HStore(327): Store=2f3f6d4a4f2ed609a76a060c63de295e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:23,088 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:19:23,089 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:19:23,094 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:19:23,105 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:23,106 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 2f3f6d4a4f2ed609a76a060c63de295e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66713592, jitterRate=-0.005890011787414551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:23,107 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 2f3f6d4a4f2ed609a76a060c63de295e: 2024-11-07T20:19:23,109 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e., pid=38, masterSystemTime=1731010763069 2024-11-07T20:19:23,113 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=2f3f6d4a4f2ed609a76a060c63de295e, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:23,113 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:19:23,113 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:19:23,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T20:19:23,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 2f3f6d4a4f2ed609a76a060c63de295e, server=15c0e1002410,32779,1731010677532 in 199 msec 2024-11-07T20:19:23,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-07T20:19:23,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2f3f6d4a4f2ed609a76a060c63de295e, ASSIGN in 358 msec 2024-11-07T20:19:23,122 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:23,123 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010763122"}]},"ts":"1731010763122"} 2024-11-07T20:19:23,124 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:23,128 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:23,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 814 msec 2024-11-07T20:19:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-07T20:19:23,321 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-07T20:19:23,321 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:23,321 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:23,327 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:23,327 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:23,327 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-07T20:19:23,330 DEBUG [hconnection-0x7a279337-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:23,332 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33652, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:23,345 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:23,351 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d01f0cf to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38d1c2a3 2024-11-07T20:19:23,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fbe7b7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:23,355 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:19:23,357 DEBUG [hconnection-0x1645d727-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:23,366 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33668, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:23,376 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:19:23,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645825 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:19:23,380 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:19:23,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645825 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:19:23,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T20:19:24,064 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-07T20:19:24,065 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-07T20:19:24,388 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T20:19:24.388Z, RpcRetryingCaller{globalStartTime=2024-11-07T20:19:23.381Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T20:19:24.388Z, RpcRetryingCaller{globalStartTime=2024-11-07T20:19:23.381Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:19:24,388 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1007 2024-11-07T20:19:24,389 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d01f0cf to 127.0.0.1:59122 2024-11-07T20:19:24,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:24,389 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22f4f96b to 127.0.0.1:59122 2024-11-07T20:19:24,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:24,390 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:24,390 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:24,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:24,390 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29caf2bf to 127.0.0.1:59122 2024-11-07T20:19:24,401 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 231) - Thread LEAK? -, OpenFileDescriptor=459 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=389 (was 379) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5676 (was 5681) 2024-11-07T20:19:24,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=5676 2024-11-07T20:19:24,415 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1dec9aae to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a11225f 2024-11-07T20:19:24,418 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x1dec9aae {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403e2eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:24,419 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5007b807 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4b26d3e1 2024-11-07T20:19:24,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e619c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:24,428 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:24,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T20:19:24,433 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:24,433 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:24,433 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-07T20:19:24,434 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T20:19:24,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741857_1033 (size=422) 2024-11-07T20:19:24,854 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7ead674e6838d105f6f8e11d652b860c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:24,882 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:19:24,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741858_1034 (size=73) 2024-11-07T20:19:25,281 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:19:25,294 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:25,294 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 7ead674e6838d105f6f8e11d652b860c, disabling compactions & flushes 2024-11-07T20:19:25,294 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:19:25,294 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:19:25,294 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. after waiting 0 ms 2024-11-07T20:19:25,294 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:19:25,294 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:19:25,294 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7ead674e6838d105f6f8e11d652b860c: 2024-11-07T20:19:25,296 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:25,297 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731010765296"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010765296"}]},"ts":"1731010765296"} 2024-11-07T20:19:25,299 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:25,300 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:25,301 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010765301"}]},"ts":"1731010765301"} 2024-11-07T20:19:25,306 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:25,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=7ead674e6838d105f6f8e11d652b860c, ASSIGN}] 2024-11-07T20:19:25,312 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=7ead674e6838d105f6f8e11d652b860c, ASSIGN 2024-11-07T20:19:25,313 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=7ead674e6838d105f6f8e11d652b860c, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T20:19:25,464 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=7ead674e6838d105f6f8e11d652b860c, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:25,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 7ead674e6838d105f6f8e11d652b860c, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:25,612 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:19:25,620 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:25,626 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:19:25,626 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 7ead674e6838d105f6f8e11d652b860c, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:25,627 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:19:25,627 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:25,627 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:19:25,627 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:19:25,635 INFO [StoreOpener-7ead674e6838d105f6f8e11d652b860c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:19:25,638 INFO [StoreOpener-7ead674e6838d105f6f8e11d652b860c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ead674e6838d105f6f8e11d652b860c columnFamilyName testFamily 2024-11-07T20:19:25,638 DEBUG [StoreOpener-7ead674e6838d105f6f8e11d652b860c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:25,639 INFO [StoreOpener-7ead674e6838d105f6f8e11d652b860c-1 {}] regionserver.HStore(327): Store=7ead674e6838d105f6f8e11d652b860c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:25,640 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:19:25,641 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:19:25,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:19:25,648 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:25,650 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 7ead674e6838d105f6f8e11d652b860c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73094207, jitterRate=0.08918856084346771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:25,651 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 7ead674e6838d105f6f8e11d652b860c: 2024-11-07T20:19:25,652 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c., pid=41, masterSystemTime=1731010765619 2024-11-07T20:19:25,655 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:19:25,655 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:19:25,656 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=7ead674e6838d105f6f8e11d652b860c, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:25,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-07T20:19:25,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 7ead674e6838d105f6f8e11d652b860c, server=15c0e1002410,32779,1731010677532 in 191 msec 2024-11-07T20:19:25,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-07T20:19:25,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=7ead674e6838d105f6f8e11d652b860c, ASSIGN in 351 msec 2024-11-07T20:19:25,664 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:25,665 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010765664"}]},"ts":"1731010765664"} 2024-11-07T20:19:25,666 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:25,670 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:25,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.2420 sec 2024-11-07T20:19:27,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-07T20:19:27,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:27,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-07T20:19:27,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:27,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-07T20:19:27,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-07T20:19:27,442 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-07T20:19:27,442 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:27,442 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:27,448 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:27,448 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:27,448 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-07T20:19:27,451 DEBUG [hconnection-0x4342dffd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:27,452 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33678, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:27,462 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:27,465 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:19:27,468 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:19:27,469 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T20:19:30,066 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-07T20:19:31,128 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:19:31,242 DEBUG [master/15c0e1002410:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T20:19:31,247 DEBUG [master/15c0e1002410:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 5dbcc8e35a32fe336d087243233e5f12 changed from -1.0 to 0.0, refreshing cache 2024-11-07T20:19:31,939 ERROR [LeaseRenewer:jenkins@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39675,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:19:33,095 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39675,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:19:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T20:19:34,492 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:19:34,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:33678 deadline: 1731010776468, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:19:34,493 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:19:34,493 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645829 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:19:34,494 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7028 2024-11-07T20:19:34,494 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5007b807 to 127.0.0.1:59122 2024-11-07T20:19:34,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:34,494 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:34,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:34,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:34,495 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1dec9aae to 127.0.0.1:59122 2024-11-07T20:19:34,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=229 (was 236), OpenFileDescriptor=450 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 389), ProcessCount=11 (was 11), AvailableMemoryMB=5667 (was 5676) 2024-11-07T20:19:34,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=229, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=5667 2024-11-07T20:19:34,513 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59f6fab7 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a366dc 2024-11-07T20:19:34,518 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x59f6fab7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9183dd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:34,519 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b45b717 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2eeec53a 2024-11-07T20:19:34,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4023b112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:34,525 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:34,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T20:19:34,527 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:34,527 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:34,527 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-07T20:19:34,528 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T20:19:34,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741859_1035 (size=428) 2024-11-07T20:19:34,939 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ab0c24b781f1e9f98f3a812c0ae403d8, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:34,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741860_1036 (size=79) 2024-11-07T20:19:35,347 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:35,347 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ab0c24b781f1e9f98f3a812c0ae403d8, disabling compactions & flushes 2024-11-07T20:19:35,347 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:19:35,347 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:19:35,347 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. after waiting 0 ms 2024-11-07T20:19:35,347 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:19:35,347 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:19:35,347 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ab0c24b781f1e9f98f3a812c0ae403d8: 2024-11-07T20:19:35,349 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:35,349 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731010775349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010775349"}]},"ts":"1731010775349"} 2024-11-07T20:19:35,351 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:35,352 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:35,352 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010775352"}]},"ts":"1731010775352"} 2024-11-07T20:19:35,354 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:35,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=ab0c24b781f1e9f98f3a812c0ae403d8, ASSIGN}] 2024-11-07T20:19:35,359 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=ab0c24b781f1e9f98f3a812c0ae403d8, ASSIGN 2024-11-07T20:19:35,360 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=ab0c24b781f1e9f98f3a812c0ae403d8, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:35,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=ab0c24b781f1e9f98f3a812c0ae403d8, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:35,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure ab0c24b781f1e9f98f3a812c0ae403d8, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T20:19:35,665 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:35,674 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:19:35,674 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => ab0c24b781f1e9f98f3a812c0ae403d8, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:35,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:19:35,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:35,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:19:35,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:19:35,677 INFO [StoreOpener-ab0c24b781f1e9f98f3a812c0ae403d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:19:35,679 INFO [StoreOpener-ab0c24b781f1e9f98f3a812c0ae403d8-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab0c24b781f1e9f98f3a812c0ae403d8 columnFamilyName testFamily 2024-11-07T20:19:35,679 DEBUG [StoreOpener-ab0c24b781f1e9f98f3a812c0ae403d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:35,680 INFO [StoreOpener-ab0c24b781f1e9f98f3a812c0ae403d8-1 {}] regionserver.HStore(327): Store=ab0c24b781f1e9f98f3a812c0ae403d8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:35,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:19:35,683 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:19:35,687 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:19:35,693 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:35,694 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened ab0c24b781f1e9f98f3a812c0ae403d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58754010, jitterRate=-0.12449702620506287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:35,695 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for ab0c24b781f1e9f98f3a812c0ae403d8: 2024-11-07T20:19:35,696 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., pid=44, masterSystemTime=1731010775665 2024-11-07T20:19:35,698 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:19:35,698 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:19:35,699 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=ab0c24b781f1e9f98f3a812c0ae403d8, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:35,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-07T20:19:35,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure ab0c24b781f1e9f98f3a812c0ae403d8, server=15c0e1002410,32779,1731010677532 in 190 msec 2024-11-07T20:19:35,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T20:19:35,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=ab0c24b781f1e9f98f3a812c0ae403d8, ASSIGN in 350 msec 2024-11-07T20:19:35,712 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:35,712 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010775712"}]},"ts":"1731010775712"} 2024-11-07T20:19:35,715 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:35,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:35,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.1930 sec 2024-11-07T20:19:36,067 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T20:19:37,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T20:19:37,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-07T20:19:37,533 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-07T20:19:37,533 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:37,533 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:37,545 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:37,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:37,546 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T20:19:37,549 DEBUG [hconnection-0x3d63759-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:37,551 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40556, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:37,562 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:37,565 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:19:37,566 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:19:37,567 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x59f6fab7 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:19:37,567 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x59f6fab7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:37,569 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40568, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:19:37,571 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:19:41,177 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:19:43,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms 2024-11-07T20:19:43,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms 2024-11-07T20:19:43,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:19:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T20:19:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:19:44,591 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:19:44,976 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905836 expired clientIPAndPort=172.17.0.2:54824, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:19:46,568 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1978ms, rpcTimeout=1975ms 2024-11-07T20:19:46,569 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1978ms, rpcTimeout=1975ms 2024-11-07T20:19:46,569 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:19:46,569 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T20:19:43.580Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms 2024-11-07T20:19:46.569Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1978ms, rpcTimeout=1975ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1978ms, rpcTimeout=1975ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1978ms, rpcTimeout=1975ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T20:19:46,570 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9004 2024-11-07T20:19:46,571 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b45b717 to 127.0.0.1:59122 2024-11-07T20:19:46,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:46,571 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:46,571 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:46,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:46,571 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59f6fab7 to 127.0.0.1:59122 2024-11-07T20:19:46,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=227 (was 229), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=373 (was 358) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5628 (was 5667) 2024-11-07T20:19:46,591 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=373, ProcessCount=11, AvailableMemoryMB=5628 2024-11-07T20:19:46,594 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f186bb5 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b09cdaf 2024-11-07T20:19:46,597 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x2f186bb5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69bf5f65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:46,599 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x419f49ed to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d8e9d4e 2024-11-07T20:19:46,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a70489, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:46,605 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:46,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-07T20:19:46,610 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:46,610 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:46,610 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-07T20:19:46,611 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T20:19:46,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741861_1037 (size=416) 2024-11-07T20:19:47,034 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => acbc84b2cbff5d121607335cd7f60cb9, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741862_1038 (size=67) 2024-11-07T20:19:47,449 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:47,449 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing acbc84b2cbff5d121607335cd7f60cb9, disabling compactions & flushes 2024-11-07T20:19:47,449 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:19:47,449 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:19:47,449 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. after waiting 0 ms 2024-11-07T20:19:47,449 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:19:47,449 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:19:47,449 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for acbc84b2cbff5d121607335cd7f60cb9: 2024-11-07T20:19:47,451 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:47,451 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731010787451"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010787451"}]},"ts":"1731010787451"} 2024-11-07T20:19:47,453 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:47,454 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:47,454 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010787454"}]},"ts":"1731010787454"} 2024-11-07T20:19:47,456 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:47,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=acbc84b2cbff5d121607335cd7f60cb9, ASSIGN}] 2024-11-07T20:19:47,461 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=acbc84b2cbff5d121607335cd7f60cb9, ASSIGN 2024-11-07T20:19:47,461 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=acbc84b2cbff5d121607335cd7f60cb9, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:47,612 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=acbc84b2cbff5d121607335cd7f60cb9, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T20:19:47,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure acbc84b2cbff5d121607335cd7f60cb9, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:47,768 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:47,775 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:19:47,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => acbc84b2cbff5d121607335cd7f60cb9, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:47,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:19:47,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:47,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:19:47,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:19:47,778 INFO [StoreOpener-acbc84b2cbff5d121607335cd7f60cb9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:19:47,781 INFO [StoreOpener-acbc84b2cbff5d121607335cd7f60cb9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acbc84b2cbff5d121607335cd7f60cb9 columnFamilyName testFamily 2024-11-07T20:19:47,781 DEBUG [StoreOpener-acbc84b2cbff5d121607335cd7f60cb9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:47,782 INFO [StoreOpener-acbc84b2cbff5d121607335cd7f60cb9-1 {}] regionserver.HStore(327): Store=acbc84b2cbff5d121607335cd7f60cb9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:47,783 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:19:47,784 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:19:47,789 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:19:47,793 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:47,794 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened acbc84b2cbff5d121607335cd7f60cb9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58728190, jitterRate=-0.12488177418708801}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:47,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for acbc84b2cbff5d121607335cd7f60cb9: 2024-11-07T20:19:47,797 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9., pid=47, masterSystemTime=1731010787768 2024-11-07T20:19:47,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:19:47,799 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:19:47,801 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=acbc84b2cbff5d121607335cd7f60cb9, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:47,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T20:19:47,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure acbc84b2cbff5d121607335cd7f60cb9, server=15c0e1002410,32779,1731010677532 in 188 msec 2024-11-07T20:19:47,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-07T20:19:47,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=acbc84b2cbff5d121607335cd7f60cb9, ASSIGN in 348 msec 2024-11-07T20:19:47,811 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:47,811 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010787811"}]},"ts":"1731010787811"} 2024-11-07T20:19:47,813 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:47,816 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:47,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.2110 sec 2024-11-07T20:19:48,070 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-07T20:19:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-07T20:19:49,617 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-07T20:19:49,617 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:49,617 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:49,622 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:49,622 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:49,622 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-07T20:19:49,625 DEBUG [hconnection-0x7f9a7da6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:49,626 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35852, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:49,636 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:49,641 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34822dfa to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31343507 2024-11-07T20:19:49,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b4ed75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:49,645 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:19:49,647 DEBUG [hconnection-0x5b87ec9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:49,648 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35862, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:49,652 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:19:49,653 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645842 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:19:49,653 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:19:49,654 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645842 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:19:49,655 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T20:19:51,091 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:19:51,983 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905834 expired clientIPAndPort=172.17.0.2:54824, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:19:53,311 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:19:55,281 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:19:55,658 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T20:19:55.658Z, RpcRetryingCaller{globalStartTime=2024-11-07T20:19:49.654Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-07T20:19:55.658Z, RpcRetryingCaller{globalStartTime=2024-11-07T20:19:49.654Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:19:55,659 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6005 2024-11-07T20:19:55,659 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34822dfa to 127.0.0.1:59122 2024-11-07T20:19:55,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:55,659 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x419f49ed to 127.0.0.1:59122 2024-11-07T20:19:55,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:55,659 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:19:55,659 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:19:55,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:19:55,660 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f186bb5 to 127.0.0.1:59122 2024-11-07T20:19:55,669 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=233 (was 227) - Thread LEAK? -, OpenFileDescriptor=451 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 373), ProcessCount=11 (was 11), AvailableMemoryMB=5603 (was 5628) 2024-11-07T20:19:55,677 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=233, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=369, ProcessCount=11, AvailableMemoryMB=5603 2024-11-07T20:19:55,679 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x158538c4 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55fa340a 2024-11-07T20:19:55,682 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x158538c4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b39ed4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:55,682 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5827630c to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@555713e1 2024-11-07T20:19:55,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a85297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:19:55,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:19:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T20:19:55,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:19:55,690 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:55,691 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-07T20:19:55,691 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:19:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T20:19:55,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741863_1039 (size=432) 2024-11-07T20:19:56,102 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c3c1af47e03021f6356fe5780cb33e13, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:19:56,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741864_1040 (size=83) 2024-11-07T20:19:56,155 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:19:56,510 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:56,511 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing c3c1af47e03021f6356fe5780cb33e13, disabling compactions & flushes 2024-11-07T20:19:56,511 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:19:56,511 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:19:56,511 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. after waiting 0 ms 2024-11-07T20:19:56,511 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:19:56,511 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:19:56,511 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for c3c1af47e03021f6356fe5780cb33e13: 2024-11-07T20:19:56,512 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:19:56,512 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731010796512"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010796512"}]},"ts":"1731010796512"} 2024-11-07T20:19:56,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:19:56,515 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:19:56,516 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010796515"}]},"ts":"1731010796515"} 2024-11-07T20:19:56,517 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-07T20:19:56,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c3c1af47e03021f6356fe5780cb33e13, ASSIGN}] 2024-11-07T20:19:56,522 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c3c1af47e03021f6356fe5780cb33e13, ASSIGN 2024-11-07T20:19:56,523 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c3c1af47e03021f6356fe5780cb33e13, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:19:56,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=c3c1af47e03021f6356fe5780cb33e13, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:56,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure c3c1af47e03021f6356fe5780cb33e13, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:19:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T20:19:56,828 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:19:56,833 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:19:56,833 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => c3c1af47e03021f6356fe5780cb33e13, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:19:56,833 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:19:56,834 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:19:56,834 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:19:56,834 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:19:56,836 INFO [StoreOpener-c3c1af47e03021f6356fe5780cb33e13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:19:56,837 INFO [StoreOpener-c3c1af47e03021f6356fe5780cb33e13-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3c1af47e03021f6356fe5780cb33e13 columnFamilyName testFamily 2024-11-07T20:19:56,837 DEBUG [StoreOpener-c3c1af47e03021f6356fe5780cb33e13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:19:56,838 INFO [StoreOpener-c3c1af47e03021f6356fe5780cb33e13-1 {}] regionserver.HStore(327): Store=c3c1af47e03021f6356fe5780cb33e13/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:19:56,839 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:19:56,839 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:19:56,842 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:19:56,844 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:19:56,845 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened c3c1af47e03021f6356fe5780cb33e13; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67515018, jitterRate=0.006052166223526001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:19:56,845 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for c3c1af47e03021f6356fe5780cb33e13: 2024-11-07T20:19:56,846 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., pid=50, masterSystemTime=1731010796828 2024-11-07T20:19:56,848 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:19:56,848 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:19:56,849 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=c3c1af47e03021f6356fe5780cb33e13, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:19:56,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-07T20:19:56,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure c3c1af47e03021f6356fe5780cb33e13, server=15c0e1002410,32779,1731010677532 in 175 msec 2024-11-07T20:19:56,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T20:19:56,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=c3c1af47e03021f6356fe5780cb33e13, ASSIGN in 332 msec 2024-11-07T20:19:56,855 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:19:56,855 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010796855"}]},"ts":"1731010796855"} 2024-11-07T20:19:56,857 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-07T20:19:56,859 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:19:56,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.1710 sec 2024-11-07T20:19:57,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T20:19:57,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:57,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-07T20:19:57,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-07T20:19:57,988 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905829 expired clientIPAndPort=172.17.0.2:48160, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:19:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-07T20:19:58,696 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-07T20:19:58,696 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-07T20:19:58,696 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:58,700 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-07T20:19:58,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:19:58,700 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-07T20:19:58,702 DEBUG [hconnection-0x4898485e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:58,703 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44384, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:58,711 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:19:58,714 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:19:58,714 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:19:58,715 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x158538c4 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:19:58,715 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x158538c4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:19:58,716 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44400, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:19:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:19:58,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:19:58,789 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905823 expired clientIPAndPort=172.17.0.2:32792, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:19:58,789 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905822 expired clientIPAndPort=172.17.0.2:32792, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:19:58,789 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905825 expired clientIPAndPort=172.17.0.2:32792, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:19:59,728 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-07T20:19:59,728 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-07T20:19:59,728 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:20:00,072 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-07T20:20:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T20:20:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:20:00,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:20:01,718 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms 2024-11-07T20:20:01,718 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms 2024-11-07T20:20:01,718 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:20:01,719 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T20:19:59.728Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-07T20:20:01.718Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=975ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T20:20:01,720 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3006 2024-11-07T20:20:01,720 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5827630c to 127.0.0.1:59122 2024-11-07T20:20:01,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:01,720 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:01,720 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:01,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:01,720 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x158538c4 to 127.0.0.1:59122 2024-11-07T20:20:01,730 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=231 (was 233), OpenFileDescriptor=448 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 369), ProcessCount=11 (was 11), AvailableMemoryMB=5600 (was 5603) 2024-11-07T20:20:01,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=5599 2024-11-07T20:20:01,741 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01467966 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4dddd3a5 2024-11-07T20:20:01,744 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x01467966 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f452bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:01,745 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01bcf3d2 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@735b6407 2024-11-07T20:20:01,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4164be43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:01,750 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T20:20:01,752 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:01,752 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:01,752 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-07T20:20:01,753 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T20:20:01,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741865_1041 (size=436) 2024-11-07T20:20:01,891 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905816 expired clientIPAndPort=172.17.0.2:50180, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:20:02,163 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => eba09a860a3a6c62a1b5d86497c767fa, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741866_1042 (size=87) 2024-11-07T20:20:02,240 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:20:02,335 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:20:02,571 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:02,572 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing eba09a860a3a6c62a1b5d86497c767fa, disabling compactions & flushes 2024-11-07T20:20:02,572 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:20:02,572 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:20:02,572 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. after waiting 0 ms 2024-11-07T20:20:02,572 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:20:02,572 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:20:02,572 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for eba09a860a3a6c62a1b5d86497c767fa: 2024-11-07T20:20:02,573 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:02,573 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1731010802573"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010802573"}]},"ts":"1731010802573"} 2024-11-07T20:20:02,575 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:02,576 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:02,576 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010802576"}]},"ts":"1731010802576"} 2024-11-07T20:20:02,577 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:02,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eba09a860a3a6c62a1b5d86497c767fa, ASSIGN}] 2024-11-07T20:20:02,587 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eba09a860a3a6c62a1b5d86497c767fa, ASSIGN 2024-11-07T20:20:02,587 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eba09a860a3a6c62a1b5d86497c767fa, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:02,738 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=eba09a860a3a6c62a1b5d86497c767fa, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:02,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure eba09a860a3a6c62a1b5d86497c767fa, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T20:20:02,892 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:02,896 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:20:02,897 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => eba09a860a3a6c62a1b5d86497c767fa, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:02,897 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:20:02,897 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:02,897 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:20:02,897 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:20:02,899 INFO [StoreOpener-eba09a860a3a6c62a1b5d86497c767fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:20:02,901 INFO [StoreOpener-eba09a860a3a6c62a1b5d86497c767fa-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eba09a860a3a6c62a1b5d86497c767fa columnFamilyName testFamily 2024-11-07T20:20:02,901 DEBUG [StoreOpener-eba09a860a3a6c62a1b5d86497c767fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:02,901 INFO [StoreOpener-eba09a860a3a6c62a1b5d86497c767fa-1 {}] regionserver.HStore(327): Store=eba09a860a3a6c62a1b5d86497c767fa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:02,902 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:20:02,903 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:20:02,905 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:20:02,907 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:02,908 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened eba09a860a3a6c62a1b5d86497c767fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59894347, jitterRate=-0.10750468075275421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:02,909 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for eba09a860a3a6c62a1b5d86497c767fa: 2024-11-07T20:20:02,910 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa., pid=53, masterSystemTime=1731010802892 2024-11-07T20:20:02,912 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:20:02,912 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:20:02,912 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=eba09a860a3a6c62a1b5d86497c767fa, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:02,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T20:20:02,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure eba09a860a3a6c62a1b5d86497c767fa, server=15c0e1002410,32779,1731010677532 in 174 msec 2024-11-07T20:20:02,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-07T20:20:02,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=eba09a860a3a6c62a1b5d86497c767fa, ASSIGN in 329 msec 2024-11-07T20:20:02,918 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:02,918 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010802918"}]},"ts":"1731010802918"} 2024-11-07T20:20:02,919 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:02,922 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:02,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.1720 sec 2024-11-07T20:20:04,534 ERROR [LeaseRenewer:jenkins@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39675,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:20:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-07T20:20:04,757 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-07T20:20:04,757 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:04,757 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:04,761 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:04,761 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:04,761 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-07T20:20:04,763 DEBUG [hconnection-0x48624264-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:04,764 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53668, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:04,772 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:04,774 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T20:20:04,774 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:20:04,775 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x01467966 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:20:04,775 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x01467966 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:04,776 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53682, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:20:04,779 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T20:20:04,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645854 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:04,780 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T20:20:04,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645854 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:04,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905762 2024-11-07T20:20:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53682 deadline: 1731010807779, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:04,781 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645854 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:04,782 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T20:20:04,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645855 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:04,783 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T20:20:04,783 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:04,784 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:04,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645856 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:04,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905760 2024-11-07T20:20:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53682 deadline: 1731010807783, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:04,785 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645856 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:04,786 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:04,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645857 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:04,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6417099302534905759 2024-11-07T20:20:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53682 deadline: 1731010807785, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:04,787 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645857 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:04,788 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:04,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645858 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:04,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6417099302534905758 2024-11-07T20:20:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:53682 deadline: 1731010807787, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:04,789 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645858 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:04,790 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:04,790 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01bcf3d2 to 127.0.0.1:59122 2024-11-07T20:20:04,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:04,790 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:04,790 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:04,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:04,790 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01467966 to 127.0.0.1:59122 2024-11-07T20:20:04,800 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=232 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=334 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=5583 (was 5599) 2024-11-07T20:20:04,808 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=232, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=334, ProcessCount=11, AvailableMemoryMB=5583 2024-11-07T20:20:04,810 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7012e8b4 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e81c4b4 2024-11-07T20:20:04,813 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x7012e8b4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ca4c05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:04,814 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x377ae836 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4eee6ba2 2024-11-07T20:20:04,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e6d83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:04,820 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:04,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T20:20:04,822 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:04,822 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:04,823 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:04,823 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-07T20:20:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T20:20:04,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741867_1043 (size=422) 2024-11-07T20:20:05,233 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2683d6dd943cbb13e85b92ced21cdb05, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:05,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741868_1044 (size=73) 2024-11-07T20:20:05,641 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:05,641 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 2683d6dd943cbb13e85b92ced21cdb05, disabling compactions & flushes 2024-11-07T20:20:05,641 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:20:05,641 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:20:05,642 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. after waiting 0 ms 2024-11-07T20:20:05,642 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:20:05,642 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:20:05,642 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2683d6dd943cbb13e85b92ced21cdb05: 2024-11-07T20:20:05,643 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:05,643 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731010805643"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010805643"}]},"ts":"1731010805643"} 2024-11-07T20:20:05,645 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:05,646 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:05,646 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010805646"}]},"ts":"1731010805646"} 2024-11-07T20:20:05,648 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:05,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=2683d6dd943cbb13e85b92ced21cdb05, ASSIGN}] 2024-11-07T20:20:05,654 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=2683d6dd943cbb13e85b92ced21cdb05, ASSIGN 2024-11-07T20:20:05,655 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=2683d6dd943cbb13e85b92ced21cdb05, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:05,695 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39675,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:20:05,805 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=2683d6dd943cbb13e85b92ced21cdb05, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:05,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure 2683d6dd943cbb13e85b92ced21cdb05, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T20:20:05,960 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:05,965 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:20:05,965 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => 2683d6dd943cbb13e85b92ced21cdb05, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:05,966 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true 2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:20:05,966 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:05,966 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for 2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:20:05,966 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for 2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:20:05,968 INFO [StoreOpener-2683d6dd943cbb13e85b92ced21cdb05-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:20:05,970 INFO [StoreOpener-2683d6dd943cbb13e85b92ced21cdb05-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2683d6dd943cbb13e85b92ced21cdb05 columnFamilyName testFamily 2024-11-07T20:20:05,970 DEBUG [StoreOpener-2683d6dd943cbb13e85b92ced21cdb05-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:05,971 INFO [StoreOpener-2683d6dd943cbb13e85b92ced21cdb05-1 {}] regionserver.HStore(327): Store=2683d6dd943cbb13e85b92ced21cdb05/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:05,972 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:20:05,973 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:20:05,976 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for 2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:20:05,981 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:05,981 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened 2683d6dd943cbb13e85b92ced21cdb05; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65115718, jitterRate=-0.02970018982887268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:05,982 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for 2683d6dd943cbb13e85b92ced21cdb05: 2024-11-07T20:20:05,983 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05., pid=56, masterSystemTime=1731010805960 2024-11-07T20:20:05,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:20:05,986 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:20:05,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=2683d6dd943cbb13e85b92ced21cdb05, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:05,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-07T20:20:05,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure 2683d6dd943cbb13e85b92ced21cdb05, server=15c0e1002410,32779,1731010677532 in 180 msec 2024-11-07T20:20:05,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T20:20:05,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=2683d6dd943cbb13e85b92ced21cdb05, ASSIGN in 338 msec 2024-11-07T20:20:05,993 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:05,993 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010805993"}]},"ts":"1731010805993"} 2024-11-07T20:20:05,995 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:05,998 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:06,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1780 sec 2024-11-07T20:20:06,073 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-07T20:20:06,074 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-07T20:20:06,495 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905811 expired clientIPAndPort=172.17.0.2:50182, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:20:07,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-07T20:20:07,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:07,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-07T20:20:07,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-07T20:20:07,828 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-07T20:20:07,828 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:07,828 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:07,832 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:07,832 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:07,832 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T20:20:07,835 DEBUG [hconnection-0x5eb758a9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:07,836 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53688, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:07,844 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:07,847 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:20:07,848 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:20:08,399 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:20:08,497 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905809 expired clientIPAndPort=172.17.0.2:50182, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:20:12,601 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905804 expired clientIPAndPort=172.17.0.2:50200, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:20:14,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T20:20:14,858 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:20:16,849 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05., hostname=15c0e1002410,32779,1731010677532, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:20:16,849 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T20:20:16.849Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11014: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1991ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:20:16,850 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9003 2024-11-07T20:20:16,852 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x377ae836 to 127.0.0.1:59122 2024-11-07T20:20:16,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:16,852 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:16,852 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:16,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:16,852 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7012e8b4 to 127.0.0.1:59122 2024-11-07T20:20:16,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=228 (was 232), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=323 (was 334), ProcessCount=11 (was 11), AvailableMemoryMB=5540 (was 5583) 2024-11-07T20:20:16,879 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=228, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=323, ProcessCount=11, AvailableMemoryMB=5539 2024-11-07T20:20:16,881 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06d82485 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@337706a4 2024-11-07T20:20:16,890 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x06d82485 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7f8413, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:16,891 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x486002f4 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5aab2672 2024-11-07T20:20:16,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d90cdae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:16,906 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:16,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T20:20:16,908 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:16,909 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:16,909 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-07T20:20:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T20:20:16,916 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741869_1045 (size=420) 2024-11-07T20:20:17,332 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b7c9dd8db5786caaf3eace20a2133768, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:17,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741870_1046 (size=71) 2024-11-07T20:20:17,342 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:17,342 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing b7c9dd8db5786caaf3eace20a2133768, disabling compactions & flushes 2024-11-07T20:20:17,342 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:20:17,342 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:20:17,342 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. after waiting 0 ms 2024-11-07T20:20:17,342 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:20:17,342 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:20:17,342 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for b7c9dd8db5786caaf3eace20a2133768: 2024-11-07T20:20:17,343 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:17,343 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731010817343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010817343"}]},"ts":"1731010817343"} 2024-11-07T20:20:17,345 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:17,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:17,347 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010817346"}]},"ts":"1731010817346"} 2024-11-07T20:20:17,348 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:17,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=b7c9dd8db5786caaf3eace20a2133768, ASSIGN}] 2024-11-07T20:20:17,354 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=b7c9dd8db5786caaf3eace20a2133768, ASSIGN 2024-11-07T20:20:17,355 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=b7c9dd8db5786caaf3eace20a2133768, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:17,505 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=b7c9dd8db5786caaf3eace20a2133768, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:17,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure b7c9dd8db5786caaf3eace20a2133768, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:17,660 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:17,667 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:20:17,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => b7c9dd8db5786caaf3eace20a2133768, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:17,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:20:17,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:17,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:20:17,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:20:17,670 INFO [StoreOpener-b7c9dd8db5786caaf3eace20a2133768-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:20:17,672 INFO [StoreOpener-b7c9dd8db5786caaf3eace20a2133768-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7c9dd8db5786caaf3eace20a2133768 columnFamilyName testFamily 2024-11-07T20:20:17,673 DEBUG [StoreOpener-b7c9dd8db5786caaf3eace20a2133768-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:17,674 INFO [StoreOpener-b7c9dd8db5786caaf3eace20a2133768-1 {}] regionserver.HStore(327): Store=b7c9dd8db5786caaf3eace20a2133768/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:17,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:20:17,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:20:17,678 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:20:17,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:17,685 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened b7c9dd8db5786caaf3eace20a2133768; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72772533, jitterRate=0.08439524471759796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:17,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for b7c9dd8db5786caaf3eace20a2133768: 2024-11-07T20:20:17,687 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768., pid=59, masterSystemTime=1731010817660 2024-11-07T20:20:17,689 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:20:17,689 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:20:17,690 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=b7c9dd8db5786caaf3eace20a2133768, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:17,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T20:20:17,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure b7c9dd8db5786caaf3eace20a2133768, server=15c0e1002410,32779,1731010677532 in 184 msec 2024-11-07T20:20:17,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T20:20:17,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=b7c9dd8db5786caaf3eace20a2133768, ASSIGN in 341 msec 2024-11-07T20:20:17,697 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:17,697 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010817697"}]},"ts":"1731010817697"} 2024-11-07T20:20:17,698 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:17,701 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:17,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 795 msec 2024-11-07T20:20:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-07T20:20:17,913 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-07T20:20:17,913 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:17,913 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:17,918 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:17,919 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:17,919 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T20:20:17,921 DEBUG [hconnection-0x469b03a7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:17,923 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34644, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:17,933 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:17,935 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:20:17,936 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:20:17,937 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x06d82485 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:20:17,937 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x06d82485 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:17,939 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34650, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:20:17,941 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:20:17,942 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645870 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:17,943 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:20:17,943 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645870 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:17,943 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T20:20:18,076 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-07T20:20:21,359 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:20:23,224 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:20:24,914 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905791 expired clientIPAndPort=172.17.0.2:33668, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:20:25,282 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:20:26,948 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9005 2024-11-07T20:20:26,948 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x486002f4 to 127.0.0.1:59122 2024-11-07T20:20:26,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:26,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:26,948 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645870 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:26,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:26,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:26,949 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x06d82485 to 127.0.0.1:59122 2024-11-07T20:20:26,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905746' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T20:20:26,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:34650 deadline: 1731010832948, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905746' does not exist 2024-11-07T20:20:26,949 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 15c0e1002410,32779,1731010677532 for closing scanner id = -6417099302534905746 for b7c9dd8db5786caaf3eace20a2133768 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T20:20:26,954 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:20:26,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 323) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5465 (was 5539) 2024-11-07T20:20:26,977 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=5464 2024-11-07T20:20:26,979 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55e91616 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41b51657 2024-11-07T20:20:26,983 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x55e91616 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f786d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:26,985 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x069fcd7b to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c6da7be 2024-11-07T20:20:26,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a95983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:26,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T20:20:26,997 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:26,997 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:26,997 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-07T20:20:26,999 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T20:20:27,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741871_1047 (size=431) 2024-11-07T20:20:27,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-07T20:20:27,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:27,424 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2416e7f18037c665ca8a4b6f6d200c32, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:27,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741872_1048 (size=82) 2024-11-07T20:20:27,435 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:27,435 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 2416e7f18037c665ca8a4b6f6d200c32, disabling compactions & flushes 2024-11-07T20:20:27,435 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:20:27,435 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:20:27,435 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. after waiting 0 ms 2024-11-07T20:20:27,435 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:20:27,435 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:20:27,436 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2416e7f18037c665ca8a4b6f6d200c32: 2024-11-07T20:20:27,437 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:27,437 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731010827437"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010827437"}]},"ts":"1731010827437"} 2024-11-07T20:20:27,439 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:27,440 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:27,440 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010827440"}]},"ts":"1731010827440"} 2024-11-07T20:20:27,442 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:27,444 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:20:27,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=2416e7f18037c665ca8a4b6f6d200c32, ASSIGN}] 2024-11-07T20:20:27,451 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=2416e7f18037c665ca8a4b6f6d200c32, ASSIGN 2024-11-07T20:20:27,452 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=2416e7f18037c665ca8a4b6f6d200c32, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:27,603 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=2416e7f18037c665ca8a4b6f6d200c32, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:27,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 2416e7f18037c665ca8a4b6f6d200c32, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:27,758 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:27,765 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:20:27,765 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 2416e7f18037c665ca8a4b6f6d200c32, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:27,766 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:20:27,766 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:27,766 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:20:27,766 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:20:27,768 INFO [StoreOpener-2416e7f18037c665ca8a4b6f6d200c32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:20:27,771 INFO [StoreOpener-2416e7f18037c665ca8a4b6f6d200c32-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2416e7f18037c665ca8a4b6f6d200c32 columnFamilyName testFamily 2024-11-07T20:20:27,771 DEBUG [StoreOpener-2416e7f18037c665ca8a4b6f6d200c32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:27,772 INFO [StoreOpener-2416e7f18037c665ca8a4b6f6d200c32-1 {}] regionserver.HStore(327): Store=2416e7f18037c665ca8a4b6f6d200c32/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:27,773 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:20:27,773 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:20:27,777 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:20:27,780 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:27,781 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 2416e7f18037c665ca8a4b6f6d200c32; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74185988, jitterRate=0.1054573655128479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:27,782 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 2416e7f18037c665ca8a4b6f6d200c32: 2024-11-07T20:20:27,783 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32., pid=62, masterSystemTime=1731010827758 2024-11-07T20:20:27,785 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:20:27,785 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:20:27,786 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=2416e7f18037c665ca8a4b6f6d200c32, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:27,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-07T20:20:27,791 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 2416e7f18037c665ca8a4b6f6d200c32, server=15c0e1002410,32779,1731010677532 in 182 msec 2024-11-07T20:20:27,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T20:20:27,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=2416e7f18037c665ca8a4b6f6d200c32, ASSIGN in 342 msec 2024-11-07T20:20:27,794 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:27,794 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010827794"}]},"ts":"1731010827794"} 2024-11-07T20:20:27,796 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:27,800 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:27,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 806 msec 2024-11-07T20:20:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-07T20:20:28,004 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-07T20:20:28,004 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:28,005 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:28,011 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:28,011 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:28,011 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-07T20:20:28,014 DEBUG [hconnection-0x3ac43334-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:28,015 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36922, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:28,025 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:28,028 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-07T20:20:28,030 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-07T20:20:28,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645874 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:28,036 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-07T20:20:28,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645874 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:28,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905742 2024-11-07T20:20:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:36922 deadline: 1731010831040, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:28,042 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:20:28,050 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-07T20:20:28,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645875 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:28,051 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-07T20:20:28,052 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:28,053 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:28,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645876 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:28,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6417099302534905740 2024-11-07T20:20:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:36922 deadline: 1731010831054, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:28,055 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:20:28,057 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:28,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645877 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:28,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6417099302534905739 2024-11-07T20:20:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:36922 deadline: 1731010831058, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:28,059 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:20:28,061 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:28,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645878 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:28,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6417099302534905738 2024-11-07T20:20:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:36922 deadline: 1731010831062, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-07T20:20:28,063 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:20:28,065 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-07T20:20:28,066 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x069fcd7b to 127.0.0.1:59122 2024-11-07T20:20:28,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:28,066 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:28,066 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:28,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:28,066 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x55e91616 to 127.0.0.1:59122 2024-11-07T20:20:28,078 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=234 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 365) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5462 (was 5464) 2024-11-07T20:20:28,089 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=5462 2024-11-07T20:20:28,093 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x146c32a6 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ed1e337 2024-11-07T20:20:28,097 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x146c32a6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6458a8be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:28,098 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x36e9ce2a to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34ec5feb 2024-11-07T20:20:28,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4160a2d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:28,116 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T20:20:28,118 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:28,118 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:28,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-07T20:20:28,119 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T20:20:28,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741873_1049 (size=425) 2024-11-07T20:20:28,532 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e0438524b59c2ac6ff13273e4babab72, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:28,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741874_1050 (size=76) 2024-11-07T20:20:28,549 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:28,549 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing e0438524b59c2ac6ff13273e4babab72, disabling compactions & flushes 2024-11-07T20:20:28,549 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:20:28,549 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:20:28,549 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. after waiting 0 ms 2024-11-07T20:20:28,549 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:20:28,549 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:20:28,549 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for e0438524b59c2ac6ff13273e4babab72: 2024-11-07T20:20:28,551 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:28,551 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731010828551"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010828551"}]},"ts":"1731010828551"} 2024-11-07T20:20:28,553 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:28,554 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:28,554 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010828554"}]},"ts":"1731010828554"} 2024-11-07T20:20:28,557 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:28,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=e0438524b59c2ac6ff13273e4babab72, ASSIGN}] 2024-11-07T20:20:28,564 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=e0438524b59c2ac6ff13273e4babab72, ASSIGN 2024-11-07T20:20:28,565 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=e0438524b59c2ac6ff13273e4babab72, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:28,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=e0438524b59c2ac6ff13273e4babab72, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:28,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure e0438524b59c2ac6ff13273e4babab72, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:28,872 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:28,877 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:20:28,877 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => e0438524b59c2ac6ff13273e4babab72, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:28,878 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:20:28,878 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:28,878 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:20:28,878 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:20:28,880 INFO [StoreOpener-e0438524b59c2ac6ff13273e4babab72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:20:28,883 INFO [StoreOpener-e0438524b59c2ac6ff13273e4babab72-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0438524b59c2ac6ff13273e4babab72 columnFamilyName testFamily 2024-11-07T20:20:28,883 DEBUG [StoreOpener-e0438524b59c2ac6ff13273e4babab72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:28,883 INFO [StoreOpener-e0438524b59c2ac6ff13273e4babab72-1 {}] regionserver.HStore(327): Store=e0438524b59c2ac6ff13273e4babab72/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:28,885 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:20:28,885 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:20:28,888 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:20:28,891 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:28,893 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened e0438524b59c2ac6ff13273e4babab72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64131988, jitterRate=-0.044358909130096436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:28,894 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for e0438524b59c2ac6ff13273e4babab72: 2024-11-07T20:20:28,895 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72., pid=65, masterSystemTime=1731010828872 2024-11-07T20:20:28,897 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:20:28,897 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:20:28,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=e0438524b59c2ac6ff13273e4babab72, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:28,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T20:20:28,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure e0438524b59c2ac6ff13273e4babab72, server=15c0e1002410,32779,1731010677532 in 180 msec 2024-11-07T20:20:28,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T20:20:28,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=e0438524b59c2ac6ff13273e4babab72, ASSIGN in 339 msec 2024-11-07T20:20:28,905 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:28,906 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010828905"}]},"ts":"1731010828905"} 2024-11-07T20:20:28,907 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:28,910 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:28,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 794 msec 2024-11-07T20:20:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-07T20:20:29,123 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-07T20:20:29,123 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:29,124 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:29,128 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:29,129 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:29,129 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T20:20:29,131 DEBUG [hconnection-0x5a4e63f9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:29,133 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36936, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:29,144 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:29,146 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:20:29,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:20:29,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-07T20:20:30,078 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-07T20:20:30,080 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-07T20:20:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T20:20:31,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:20:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:36936 deadline: 1731010832150, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:20:31,173 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:20:31,173 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645883 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:20:31,174 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2027 2024-11-07T20:20:31,174 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x36e9ce2a to 127.0.0.1:59122 2024-11-07T20:20:31,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:31,175 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:31,175 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:31,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:31,175 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x146c32a6 to 127.0.0.1:59122 2024-11-07T20:20:31,184 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=234 (was 234), OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=5529 (was 5462) - AvailableMemoryMB LEAK? - 2024-11-07T20:20:31,193 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=234, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=5528 2024-11-07T20:20:31,194 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x33672bde to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10062059 2024-11-07T20:20:31,198 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x33672bde {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d720651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:31,198 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a74f4f4 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61ecb5e3 2024-11-07T20:20:31,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdf4bdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:31,204 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T20:20:31,206 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:31,206 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:31,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-07T20:20:31,207 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T20:20:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741875_1051 (size=426) 2024-11-07T20:20:31,617 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c20367372a9444f76086b80bb0cd5335, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:31,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741876_1052 (size=77) 2024-11-07T20:20:32,026 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:32,026 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing c20367372a9444f76086b80bb0cd5335, disabling compactions & flushes 2024-11-07T20:20:32,026 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:20:32,026 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:20:32,026 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. after waiting 0 ms 2024-11-07T20:20:32,026 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:20:32,026 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:20:32,026 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for c20367372a9444f76086b80bb0cd5335: 2024-11-07T20:20:32,027 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:32,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731010832027"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010832027"}]},"ts":"1731010832027"} 2024-11-07T20:20:32,029 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:32,030 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:32,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010832030"}]},"ts":"1731010832030"} 2024-11-07T20:20:32,032 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:32,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=c20367372a9444f76086b80bb0cd5335, ASSIGN}] 2024-11-07T20:20:32,037 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=c20367372a9444f76086b80bb0cd5335, ASSIGN 2024-11-07T20:20:32,038 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=c20367372a9444f76086b80bb0cd5335, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:32,189 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=c20367372a9444f76086b80bb0cd5335, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:32,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure c20367372a9444f76086b80bb0cd5335, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T20:20:32,343 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:32,347 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:20:32,348 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => c20367372a9444f76086b80bb0cd5335, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:32,348 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true c20367372a9444f76086b80bb0cd5335 2024-11-07T20:20:32,348 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:32,348 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for c20367372a9444f76086b80bb0cd5335 2024-11-07T20:20:32,348 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for c20367372a9444f76086b80bb0cd5335 2024-11-07T20:20:32,350 INFO [StoreOpener-c20367372a9444f76086b80bb0cd5335-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c20367372a9444f76086b80bb0cd5335 2024-11-07T20:20:32,352 INFO [StoreOpener-c20367372a9444f76086b80bb0cd5335-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c20367372a9444f76086b80bb0cd5335 columnFamilyName testFamily 2024-11-07T20:20:32,352 DEBUG [StoreOpener-c20367372a9444f76086b80bb0cd5335-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:32,352 INFO [StoreOpener-c20367372a9444f76086b80bb0cd5335-1 {}] regionserver.HStore(327): Store=c20367372a9444f76086b80bb0cd5335/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:32,353 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335 2024-11-07T20:20:32,353 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335 2024-11-07T20:20:32,356 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for c20367372a9444f76086b80bb0cd5335 2024-11-07T20:20:32,358 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:32,359 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened c20367372a9444f76086b80bb0cd5335; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59441323, jitterRate=-0.11425526440143585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:32,360 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for c20367372a9444f76086b80bb0cd5335: 2024-11-07T20:20:32,360 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., pid=68, masterSystemTime=1731010832342 2024-11-07T20:20:32,362 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:20:32,362 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:20:32,362 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=c20367372a9444f76086b80bb0cd5335, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:32,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-07T20:20:32,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure c20367372a9444f76086b80bb0cd5335, server=15c0e1002410,32779,1731010677532 in 173 msec 2024-11-07T20:20:32,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T20:20:32,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=c20367372a9444f76086b80bb0cd5335, ASSIGN in 328 msec 2024-11-07T20:20:32,367 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:32,367 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010832367"}]},"ts":"1731010832367"} 2024-11-07T20:20:32,369 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:32,371 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:32,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1670 sec 2024-11-07T20:20:33,267 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:20:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-07T20:20:34,211 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-07T20:20:34,211 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:34,212 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:34,215 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:34,216 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:34,216 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-07T20:20:34,218 DEBUG [hconnection-0x5273c870-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:34,219 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58750, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:34,227 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:34,229 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:20:34,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:20:34,524 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905787 expired clientIPAndPort=172.17.0.2:33678, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:20:34,829 ERROR [LeaseRenewer:jenkins@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39675,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:20:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-07T20:20:35,983 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39675,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:20:36,081 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-07T20:20:36,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:20:37,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T20:20:37,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:37,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-07T20:20:37,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:37,043 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T20:20:37,043 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:37,239 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., hostname=15c0e1002410,32779,1731010677532, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:20:37,239 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-07T20:20:37.239Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=992ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:20:37,240 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3011 2024-11-07T20:20:37,240 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2a74f4f4 to 127.0.0.1:59122 2024-11-07T20:20:37,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:37,240 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:37,241 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:37,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:37,241 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x33672bde to 127.0.0.1:59122 2024-11-07T20:20:37,250 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=230 (was 234), OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=5461 (was 5528) 2024-11-07T20:20:37,259 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=5460 2024-11-07T20:20:37,260 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4044be19 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@689afe29 2024-11-07T20:20:37,264 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x4044be19 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735a84b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:37,264 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4275ec03 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@de35236 2024-11-07T20:20:37,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683291e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:37,270 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T20:20:37,272 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:37,272 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:37,272 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-07T20:20:37,273 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T20:20:37,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741877_1053 (size=424) 2024-11-07T20:20:37,683 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e176f56c01b42e1dc5198946a56951a2, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741878_1054 (size=75) 2024-11-07T20:20:37,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:37,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing e176f56c01b42e1dc5198946a56951a2, disabling compactions & flushes 2024-11-07T20:20:37,692 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:20:37,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:20:37,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. after waiting 0 ms 2024-11-07T20:20:37,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:20:37,692 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:20:37,692 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for e176f56c01b42e1dc5198946a56951a2: 2024-11-07T20:20:37,693 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:37,694 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731010837693"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010837693"}]},"ts":"1731010837693"} 2024-11-07T20:20:37,695 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:37,696 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:37,696 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010837696"}]},"ts":"1731010837696"} 2024-11-07T20:20:37,698 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:37,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=e176f56c01b42e1dc5198946a56951a2, ASSIGN}] 2024-11-07T20:20:37,708 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=e176f56c01b42e1dc5198946a56951a2, ASSIGN 2024-11-07T20:20:37,709 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=e176f56c01b42e1dc5198946a56951a2, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:37,747 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:20:37,859 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=e176f56c01b42e1dc5198946a56951a2, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:37,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure e176f56c01b42e1dc5198946a56951a2, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:38,013 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:38,018 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:20:38,018 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => e176f56c01b42e1dc5198946a56951a2, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:38,019 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:20:38,019 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:38,019 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:20:38,019 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:20:38,021 INFO [StoreOpener-e176f56c01b42e1dc5198946a56951a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:20:38,023 INFO [StoreOpener-e176f56c01b42e1dc5198946a56951a2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e176f56c01b42e1dc5198946a56951a2 columnFamilyName testFamily 2024-11-07T20:20:38,023 DEBUG [StoreOpener-e176f56c01b42e1dc5198946a56951a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:38,023 INFO [StoreOpener-e176f56c01b42e1dc5198946a56951a2-1 {}] regionserver.HStore(327): Store=e176f56c01b42e1dc5198946a56951a2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:38,024 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:20:38,025 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:20:38,027 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:20:38,030 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:38,031 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened e176f56c01b42e1dc5198946a56951a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68562558, jitterRate=0.021661728620529175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:38,031 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for e176f56c01b42e1dc5198946a56951a2: 2024-11-07T20:20:38,032 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2., pid=71, masterSystemTime=1731010838013 2024-11-07T20:20:38,034 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:20:38,034 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:20:38,034 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=e176f56c01b42e1dc5198946a56951a2, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:38,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T20:20:38,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure e176f56c01b42e1dc5198946a56951a2, server=15c0e1002410,32779,1731010677532 in 175 msec 2024-11-07T20:20:38,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-07T20:20:38,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=e176f56c01b42e1dc5198946a56951a2, ASSIGN in 330 msec 2024-11-07T20:20:38,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:38,039 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010838039"}]},"ts":"1731010838039"} 2024-11-07T20:20:38,041 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:38,043 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:38,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 773 msec 2024-11-07T20:20:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-07T20:20:38,275 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-07T20:20:38,275 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:38,276 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:38,279 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:38,279 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:38,279 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-07T20:20:38,281 DEBUG [hconnection-0x513a3c03-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:38,282 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58762, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:38,290 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:38,292 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:20:38,292 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:20:38,293 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x4044be19 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:20:38,293 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x4044be19 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:38,294 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58772, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:20:38,297 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:20:38,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645895 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:38,297 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:20:38,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645895 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:38,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T20:20:41,298 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3000 2024-11-07T20:20:41,298 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4275ec03 to 127.0.0.1:59122 2024-11-07T20:20:41,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:41,298 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:41,298 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645895 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:41,298 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:41,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:41,299 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4044be19 to 127.0.0.1:59122 2024-11-07T20:20:41,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905721' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-07T20:20:41,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:58772 deadline: 1731010842298, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6417099302534905721' does not exist 2024-11-07T20:20:41,301 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:20:41,299 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 15c0e1002410,32779,1731010677532 for closing scanner id = -6417099302534905721 for e176f56c01b42e1dc5198946a56951a2 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-07T20:20:41,308 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=232 (was 230) - Thread LEAK? -, OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 353), ProcessCount=11 (was 11), AvailableMemoryMB=5454 (was 5460) 2024-11-07T20:20:41,317 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=232, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=5453 2024-11-07T20:20:41,319 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x160d069d to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f92b4da 2024-11-07T20:20:41,323 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x160d069d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369f4c45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:41,324 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17444b98 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61dd708b 2024-11-07T20:20:41,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9aabc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:41,330 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:41,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-07T20:20:41,332 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:41,332 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:41,332 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-07T20:20:41,333 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T20:20:41,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741879_1055 (size=416) 2024-11-07T20:20:41,743 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 86e819bb0a1f1ab205ac418e88f4cc7b, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:41,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741880_1056 (size=67) 2024-11-07T20:20:41,798 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:20:42,082 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-07T20:20:42,151 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:42,151 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 86e819bb0a1f1ab205ac418e88f4cc7b, disabling compactions & flushes 2024-11-07T20:20:42,151 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:20:42,151 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:20:42,151 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. after waiting 0 ms 2024-11-07T20:20:42,151 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:20:42,151 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:20:42,151 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 86e819bb0a1f1ab205ac418e88f4cc7b: 2024-11-07T20:20:42,152 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:42,152 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731010842152"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010842152"}]},"ts":"1731010842152"} 2024-11-07T20:20:42,154 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:42,155 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:42,155 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010842155"}]},"ts":"1731010842155"} 2024-11-07T20:20:42,156 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:42,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=86e819bb0a1f1ab205ac418e88f4cc7b, ASSIGN}] 2024-11-07T20:20:42,160 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=86e819bb0a1f1ab205ac418e88f4cc7b, ASSIGN 2024-11-07T20:20:42,161 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=86e819bb0a1f1ab205ac418e88f4cc7b, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:42,312 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=86e819bb0a1f1ab205ac418e88f4cc7b, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:42,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 86e819bb0a1f1ab205ac418e88f4cc7b, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T20:20:42,466 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:42,470 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:20:42,471 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 86e819bb0a1f1ab205ac418e88f4cc7b, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:42,471 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:20:42,471 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:42,471 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:20:42,471 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:20:42,473 INFO [StoreOpener-86e819bb0a1f1ab205ac418e88f4cc7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:20:42,475 INFO [StoreOpener-86e819bb0a1f1ab205ac418e88f4cc7b-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86e819bb0a1f1ab205ac418e88f4cc7b columnFamilyName testFamily 2024-11-07T20:20:42,475 DEBUG [StoreOpener-86e819bb0a1f1ab205ac418e88f4cc7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:42,475 INFO [StoreOpener-86e819bb0a1f1ab205ac418e88f4cc7b-1 {}] regionserver.HStore(327): Store=86e819bb0a1f1ab205ac418e88f4cc7b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:42,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:20:42,477 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:20:42,479 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:20:42,481 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:42,482 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 86e819bb0a1f1ab205ac418e88f4cc7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70289521, jitterRate=0.047395482659339905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:42,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 86e819bb0a1f1ab205ac418e88f4cc7b: 2024-11-07T20:20:42,484 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b., pid=74, masterSystemTime=1731010842466 2024-11-07T20:20:42,485 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:20:42,486 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:20:42,486 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=86e819bb0a1f1ab205ac418e88f4cc7b, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:42,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-07T20:20:42,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 86e819bb0a1f1ab205ac418e88f4cc7b, server=15c0e1002410,32779,1731010677532 in 173 msec 2024-11-07T20:20:42,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T20:20:42,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=86e819bb0a1f1ab205ac418e88f4cc7b, ASSIGN in 329 msec 2024-11-07T20:20:42,491 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:42,491 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010842491"}]},"ts":"1731010842491"} 2024-11-07T20:20:42,492 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:42,495 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:42,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.1650 sec 2024-11-07T20:20:43,520 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:20:44,132 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905781 expired clientIPAndPort=172.17.0.2:40568, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:20:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-07T20:20:44,337 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-07T20:20:44,337 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:44,338 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:44,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-07T20:20:44,346 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645896 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:44,347 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:44,347 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:44,347 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-07T20:20:44,352 DEBUG [hconnection-0x52968aac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:44,354 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47006, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:44,363 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:44,366 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:20:44,368 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645899 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:44,368 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-07T20:20:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-07T20:20:47,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:47,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-07T20:20:47,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:48,084 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-07T20:20:50,869 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6503 2024-11-07T20:20:50,869 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17444b98 to 127.0.0.1:59122 2024-11-07T20:20:50,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:50,870 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:50,870 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:50,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:50,870 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x160d069d to 127.0.0.1:59122 2024-11-07T20:20:50,882 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=231 (was 232), OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=305 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=6275 (was 5453) - AvailableMemoryMB LEAK? - 2024-11-07T20:20:50,892 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=231, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=305, ProcessCount=11, AvailableMemoryMB=6274 2024-11-07T20:20:50,894 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6782cdc0 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36af503d 2024-11-07T20:20:50,898 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6782cdc0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@987b5b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:50,899 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46b7b31c to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4561e422 2024-11-07T20:20:50,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347cdd83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:50,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-07T20:20:50,906 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:50,906 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:50,906 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-07T20:20:50,907 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T20:20:50,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741881_1057 (size=420) 2024-11-07T20:20:51,138 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905779 expired clientIPAndPort=172.17.0.2:40568, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:20:51,317 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7a14658215f07a3fd882c2a428ad1d0d, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:51,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741882_1058 (size=71) 2024-11-07T20:20:51,725 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:51,725 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 7a14658215f07a3fd882c2a428ad1d0d, disabling compactions & flushes 2024-11-07T20:20:51,725 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:20:51,725 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:20:51,725 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. after waiting 0 ms 2024-11-07T20:20:51,725 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:20:51,725 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:20:51,725 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7a14658215f07a3fd882c2a428ad1d0d: 2024-11-07T20:20:51,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:51,726 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731010851726"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010851726"}]},"ts":"1731010851726"} 2024-11-07T20:20:51,728 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:51,729 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:51,729 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010851729"}]},"ts":"1731010851729"} 2024-11-07T20:20:51,730 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:51,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a14658215f07a3fd882c2a428ad1d0d, ASSIGN}] 2024-11-07T20:20:51,736 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a14658215f07a3fd882c2a428ad1d0d, ASSIGN 2024-11-07T20:20:51,737 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a14658215f07a3fd882c2a428ad1d0d, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:51,887 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=7a14658215f07a3fd882c2a428ad1d0d, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:51,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 7a14658215f07a3fd882c2a428ad1d0d, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T20:20:52,041 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:52,046 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:20:52,046 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => 7a14658215f07a3fd882c2a428ad1d0d, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:52,047 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:20:52,047 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:52,047 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for 7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:20:52,047 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for 7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:20:52,048 INFO [StoreOpener-7a14658215f07a3fd882c2a428ad1d0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:20:52,050 INFO [StoreOpener-7a14658215f07a3fd882c2a428ad1d0d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a14658215f07a3fd882c2a428ad1d0d columnFamilyName testFamily 2024-11-07T20:20:52,050 DEBUG [StoreOpener-7a14658215f07a3fd882c2a428ad1d0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:52,051 INFO [StoreOpener-7a14658215f07a3fd882c2a428ad1d0d-1 {}] regionserver.HStore(327): Store=7a14658215f07a3fd882c2a428ad1d0d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:52,052 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:20:52,052 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:20:52,054 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for 7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:20:52,056 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:52,057 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened 7a14658215f07a3fd882c2a428ad1d0d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70942011, jitterRate=0.05711834132671356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:52,058 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for 7a14658215f07a3fd882c2a428ad1d0d: 2024-11-07T20:20:52,059 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d., pid=77, masterSystemTime=1731010852041 2024-11-07T20:20:52,060 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:20:52,060 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:20:52,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=7a14658215f07a3fd882c2a428ad1d0d, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:52,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T20:20:52,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 7a14658215f07a3fd882c2a428ad1d0d, server=15c0e1002410,32779,1731010677532 in 172 msec 2024-11-07T20:20:52,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-07T20:20:52,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=7a14658215f07a3fd882c2a428ad1d0d, ASSIGN in 329 msec 2024-11-07T20:20:52,066 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:52,066 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010852066"}]},"ts":"1731010852066"} 2024-11-07T20:20:52,068 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:52,070 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:52,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1660 sec 2024-11-07T20:20:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-07T20:20:53,912 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-07T20:20:53,912 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:53,912 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:53,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-07T20:20:53,919 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645900 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:53,920 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:53,920 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:53,920 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-07T20:20:53,923 DEBUG [hconnection-0x117a22e8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:53,926 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48916, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:53,934 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:53,936 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:20:53,938 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645903 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-07T20:20:53,939 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-07T20:20:54,085 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-07T20:20:55,282 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:20:55,439 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-07T20:20:55,440 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x46b7b31c to 127.0.0.1:59122 2024-11-07T20:20:55,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:55,440 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:20:55,440 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:20:55,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:20:55,440 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6782cdc0 to 127.0.0.1:59122 2024-11-07T20:20:55,450 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=289 (was 305), ProcessCount=11 (was 11), AvailableMemoryMB=6148 (was 6274) 2024-11-07T20:20:55,459 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=6145 2024-11-07T20:20:55,461 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x28c20935 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55ac4d45 2024-11-07T20:20:55,465 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x28c20935 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57a7eeda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:55,466 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x512565d3 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41387cf4 2024-11-07T20:20:55,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@790b4e47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:20:55,472 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:20:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-07T20:20:55,474 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:20:55,474 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:55,474 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-07T20:20:55,475 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:20:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T20:20:55,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741883_1059 (size=419) 2024-11-07T20:20:55,884 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f49676cac4562adaefc259527bfa03c5, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:20:55,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741884_1060 (size=70) 2024-11-07T20:20:56,244 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905774 expired clientIPAndPort=172.17.0.2:35862, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:20:56,292 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:56,292 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f49676cac4562adaefc259527bfa03c5, disabling compactions & flushes 2024-11-07T20:20:56,293 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:20:56,293 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:20:56,293 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. after waiting 0 ms 2024-11-07T20:20:56,293 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:20:56,293 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:20:56,293 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f49676cac4562adaefc259527bfa03c5: 2024-11-07T20:20:56,294 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:20:56,294 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731010856294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010856294"}]},"ts":"1731010856294"} 2024-11-07T20:20:56,296 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:20:56,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:20:56,297 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010856297"}]},"ts":"1731010856297"} 2024-11-07T20:20:56,298 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:20:56,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=f49676cac4562adaefc259527bfa03c5, ASSIGN}] 2024-11-07T20:20:56,303 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=f49676cac4562adaefc259527bfa03c5, ASSIGN 2024-11-07T20:20:56,303 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=f49676cac4562adaefc259527bfa03c5, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:20:56,454 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=f49676cac4562adaefc259527bfa03c5, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:56,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure f49676cac4562adaefc259527bfa03c5, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:20:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T20:20:56,608 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:20:56,613 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:20:56,613 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => f49676cac4562adaefc259527bfa03c5, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:20:56,614 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true f49676cac4562adaefc259527bfa03c5 2024-11-07T20:20:56,614 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:20:56,614 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for f49676cac4562adaefc259527bfa03c5 2024-11-07T20:20:56,614 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for f49676cac4562adaefc259527bfa03c5 2024-11-07T20:20:56,616 INFO [StoreOpener-f49676cac4562adaefc259527bfa03c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f49676cac4562adaefc259527bfa03c5 2024-11-07T20:20:56,618 INFO [StoreOpener-f49676cac4562adaefc259527bfa03c5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f49676cac4562adaefc259527bfa03c5 columnFamilyName testFamily 2024-11-07T20:20:56,618 DEBUG [StoreOpener-f49676cac4562adaefc259527bfa03c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:20:56,618 INFO [StoreOpener-f49676cac4562adaefc259527bfa03c5-1 {}] regionserver.HStore(327): Store=f49676cac4562adaefc259527bfa03c5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:20:56,619 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5 2024-11-07T20:20:56,620 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5 2024-11-07T20:20:56,622 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for f49676cac4562adaefc259527bfa03c5 2024-11-07T20:20:56,625 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:20:56,625 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened f49676cac4562adaefc259527bfa03c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68951123, jitterRate=0.02745179831981659}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:20:56,626 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for f49676cac4562adaefc259527bfa03c5: 2024-11-07T20:20:56,627 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5., pid=80, masterSystemTime=1731010856608 2024-11-07T20:20:56,629 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:20:56,629 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:20:56,629 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=f49676cac4562adaefc259527bfa03c5, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:20:56,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-07T20:20:56,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure f49676cac4562adaefc259527bfa03c5, server=15c0e1002410,32779,1731010677532 in 175 msec 2024-11-07T20:20:56,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T20:20:56,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=f49676cac4562adaefc259527bfa03c5, ASSIGN in 330 msec 2024-11-07T20:20:56,634 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:20:56,634 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010856634"}]},"ts":"1731010856634"} 2024-11-07T20:20:56,636 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:20:56,638 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:20:56,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1660 sec 2024-11-07T20:20:57,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T20:20:57,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:57,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-07T20:20:57,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:20:57,606 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:20:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-07T20:20:58,478 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-07T20:20:58,478 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:20:58,478 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:58,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-07T20:20:58,482 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645904 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:58,483 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T20:20:58,483 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:20:58,484 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-07T20:20:58,485 DEBUG [hconnection-0xcfd39d0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:20:58,486 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48924, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:20:58,494 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:20:58,496 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:20:58,497 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:20:58,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645907 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:58,499 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:20:58,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645907 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:20:58,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-07T20:21:00,087 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-07T20:21:00,248 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905769 expired clientIPAndPort=172.17.0.2:44400, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:21:01,508 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4011: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5., hostname=15c0e1002410,32779,1731010677532, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4011: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3009ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:21:01,509 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3010 2024-11-07T20:21:01,509 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x512565d3 to 127.0.0.1:59122 2024-11-07T20:21:01,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:01,509 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:21:01,509 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:21:01,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:01,509 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x28c20935 to 127.0.0.1:59122 2024-11-07T20:21:01,518 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=230 (was 233), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 289) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6104 (was 6145) 2024-11-07T20:21:01,526 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=6104 2024-11-07T20:21:01,528 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7675fd68 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3319f4ca 2024-11-07T20:21:01,530 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x7675fd68 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e722a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:01,531 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31d9b463 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d1d3184 2024-11-07T20:21:01,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745a07f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:01,539 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:21:01,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T20:21:01,541 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:21:01,541 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:01,541 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-07T20:21:01,541 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:21:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T20:21:01,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741885_1061 (size=421) 2024-11-07T20:21:01,951 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => eb505c3486a2bcb859f8a689ce4a1422, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:21:01,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741886_1062 (size=72) 2024-11-07T20:21:02,000 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:21:02,250 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905767 expired clientIPAndPort=172.17.0.2:44400, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:21:02,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:02,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing eb505c3486a2bcb859f8a689ce4a1422, disabling compactions & flushes 2024-11-07T20:21:02,359 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:02,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:02,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. after waiting 0 ms 2024-11-07T20:21:02,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:02,359 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:02,359 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for eb505c3486a2bcb859f8a689ce4a1422: 2024-11-07T20:21:02,361 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:21:02,361 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731010862361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010862361"}]},"ts":"1731010862361"} 2024-11-07T20:21:02,363 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:21:02,363 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:21:02,364 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010862363"}]},"ts":"1731010862363"} 2024-11-07T20:21:02,365 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:21:02,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=eb505c3486a2bcb859f8a689ce4a1422, ASSIGN}] 2024-11-07T20:21:02,369 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=eb505c3486a2bcb859f8a689ce4a1422, ASSIGN 2024-11-07T20:21:02,370 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=eb505c3486a2bcb859f8a689ce4a1422, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:21:02,521 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=eb505c3486a2bcb859f8a689ce4a1422, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:02,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure eb505c3486a2bcb859f8a689ce4a1422, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:21:02,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T20:21:02,675 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:21:02,680 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:02,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => eb505c3486a2bcb859f8a689ce4a1422, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:21:02,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:02,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:02,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:02,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:02,684 INFO [StoreOpener-eb505c3486a2bcb859f8a689ce4a1422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:02,686 INFO [StoreOpener-eb505c3486a2bcb859f8a689ce4a1422-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb505c3486a2bcb859f8a689ce4a1422 columnFamilyName testFamily 2024-11-07T20:21:02,686 DEBUG [StoreOpener-eb505c3486a2bcb859f8a689ce4a1422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:02,687 INFO [StoreOpener-eb505c3486a2bcb859f8a689ce4a1422-1 {}] regionserver.HStore(327): Store=eb505c3486a2bcb859f8a689ce4a1422/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:21:02,688 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:02,688 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:02,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:02,694 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:21:02,694 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened eb505c3486a2bcb859f8a689ce4a1422; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66703475, jitterRate=-0.0060407668352127075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:21:02,696 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for eb505c3486a2bcb859f8a689ce4a1422: 2024-11-07T20:21:02,697 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422., pid=83, masterSystemTime=1731010862674 2024-11-07T20:21:02,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:02,699 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:02,699 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=eb505c3486a2bcb859f8a689ce4a1422, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:02,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T20:21:02,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure eb505c3486a2bcb859f8a689ce4a1422, server=15c0e1002410,32779,1731010677532 in 178 msec 2024-11-07T20:21:02,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-07T20:21:02,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=eb505c3486a2bcb859f8a689ce4a1422, ASSIGN in 334 msec 2024-11-07T20:21:02,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:21:02,705 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010862705"}]},"ts":"1731010862705"} 2024-11-07T20:21:02,706 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:21:02,709 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:21:02,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1700 sec 2024-11-07T20:21:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-07T20:21:04,545 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-07T20:21:04,545 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:21:04,545 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:04,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-07T20:21:04,549 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645908 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:21:04,551 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-07T20:21:04,551 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:04,551 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-07T20:21:04,553 DEBUG [hconnection-0x54429e9d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:21:04,555 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47924, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:21:04,562 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:21:04,564 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:21:04,566 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:21:04,566 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-07T20:21:06,089 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-07T20:21:07,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-07T20:21:07,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:21:07,278 ERROR [LeaseRenewer:jenkins@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39675,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:21:08,032 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39675,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:21:08,225 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:21:11,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-07T20:21:11,580 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:21:11,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:47924 deadline: 1731010873566, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-07T20:21:11,581 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-07T20:21:11,581 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 12029644771174645911 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T20:21:11,582 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7017 2024-11-07T20:21:11,582 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31d9b463 to 127.0.0.1:59122 2024-11-07T20:21:11,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:11,582 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:21:11,583 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:21:11,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:11,583 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7675fd68 to 127.0.0.1:59122 2024-11-07T20:21:11,592 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=228 (was 230), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 298), ProcessCount=11 (was 11), AvailableMemoryMB=6065 (was 6104) 2024-11-07T20:21:11,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=228, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=282, ProcessCount=11, AvailableMemoryMB=6064 2024-11-07T20:21:11,602 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35edb7ba to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@154c1bac 2024-11-07T20:21:11,604 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x35edb7ba {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392880c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:11,606 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x18d16993 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8825c2c 2024-11-07T20:21:11,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500a6ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:11,636 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:21:11,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T20:21:11,638 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:21:11,638 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:11,639 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-07T20:21:11,639 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:21:11,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T20:21:11,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741887_1063 (size=427) 2024-11-07T20:21:12,051 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b65a37ef0d404205cc6dcabe718de758, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:21:12,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741888_1064 (size=78) 2024-11-07T20:21:12,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:12,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing b65a37ef0d404205cc6dcabe718de758, disabling compactions & flushes 2024-11-07T20:21:12,459 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:12,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:12,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. after waiting 0 ms 2024-11-07T20:21:12,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:12,459 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:12,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for b65a37ef0d404205cc6dcabe718de758: 2024-11-07T20:21:12,460 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:21:12,461 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731010872460"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010872460"}]},"ts":"1731010872460"} 2024-11-07T20:21:12,462 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:21:12,463 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:21:12,463 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010872463"}]},"ts":"1731010872463"} 2024-11-07T20:21:12,464 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:21:12,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=b65a37ef0d404205cc6dcabe718de758, ASSIGN}] 2024-11-07T20:21:12,469 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=b65a37ef0d404205cc6dcabe718de758, ASSIGN 2024-11-07T20:21:12,470 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=b65a37ef0d404205cc6dcabe718de758, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:21:12,620 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=b65a37ef0d404205cc6dcabe718de758, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:12,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure b65a37ef0d404205cc6dcabe718de758, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:21:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T20:21:12,775 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:21:12,780 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:12,780 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => b65a37ef0d404205cc6dcabe718de758, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:21:12,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:12,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:12,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:12,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:12,783 INFO [StoreOpener-b65a37ef0d404205cc6dcabe718de758-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:12,784 INFO [StoreOpener-b65a37ef0d404205cc6dcabe718de758-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b65a37ef0d404205cc6dcabe718de758 columnFamilyName testFamily 2024-11-07T20:21:12,784 DEBUG [StoreOpener-b65a37ef0d404205cc6dcabe718de758-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:12,785 INFO [StoreOpener-b65a37ef0d404205cc6dcabe718de758-1 {}] regionserver.HStore(327): Store=b65a37ef0d404205cc6dcabe718de758/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:21:12,786 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:12,786 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:12,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:12,791 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:21:12,791 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened b65a37ef0d404205cc6dcabe718de758; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73916152, jitterRate=0.10143649578094482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:21:12,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for b65a37ef0d404205cc6dcabe718de758: 2024-11-07T20:21:12,792 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., pid=86, masterSystemTime=1731010872774 2024-11-07T20:21:12,794 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:12,794 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:12,795 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=b65a37ef0d404205cc6dcabe718de758, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:12,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-07T20:21:12,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure b65a37ef0d404205cc6dcabe718de758, server=15c0e1002410,32779,1731010677532 in 173 msec 2024-11-07T20:21:12,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T20:21:12,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=b65a37ef0d404205cc6dcabe718de758, ASSIGN in 330 msec 2024-11-07T20:21:12,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:21:12,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010872800"}]},"ts":"1731010872800"} 2024-11-07T20:21:12,801 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:21:12,804 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:21:12,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1670 sec 2024-11-07T20:21:14,360 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905753 expired clientIPAndPort=172.17.0.2:53688, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:21:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-07T20:21:14,643 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-07T20:21:14,643 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:21:14,643 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:14,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-07T20:21:14,647 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645913 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:21:14,648 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T20:21:14,648 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:14,648 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T20:21:14,650 DEBUG [hconnection-0x7e2909c9-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:21:14,651 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32796, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:21:14,658 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:21:14,660 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:21:14,660 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:21:14,661 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x35edb7ba {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:21:14,661 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x35edb7ba {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:21:14,662 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32810, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:21:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:21:14,664 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:21:17,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T20:21:17,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:21:18,282 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:21:18,320 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T20:21:20,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-07T20:21:20,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-07T20:21:20,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:21:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T20:21:21,366 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905751 expired clientIPAndPort=172.17.0.2:53688, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:21:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:21:21,689 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-07T20:21:23,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms 2024-11-07T20:21:23,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms 2024-11-07T20:21:23,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:21:23,668 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T20:21:20.668Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-07T20:21:23.668Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T20:21:23,670 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9010 2024-11-07T20:21:23,670 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x18d16993 to 127.0.0.1:59122 2024-11-07T20:21:23,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:23,670 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:21:23,670 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:21:23,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:23,670 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x35edb7ba to 127.0.0.1:59122 2024-11-07T20:21:23,680 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=228 (was 228), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 282), ProcessCount=11 (was 11), AvailableMemoryMB=5848 (was 6064) 2024-11-07T20:21:23,688 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=228, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=5848 2024-11-07T20:21:23,689 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d9612bd to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@170923 2024-11-07T20:21:23,695 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x1d9612bd {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7eee4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:23,695 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x760de6d1 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@311fe5ff 2024-11-07T20:21:23,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c311a9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:23,701 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:21:23,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-07T20:21:23,702 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:21:23,703 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:23,703 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-07T20:21:23,703 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:21:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T20:21:23,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741889_1065 (size=415) 2024-11-07T20:21:24,114 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 80eec78f120d1efdd45815dfc102e429, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:21:24,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741890_1066 (size=66) 2024-11-07T20:21:24,521 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:24,521 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 80eec78f120d1efdd45815dfc102e429, disabling compactions & flushes 2024-11-07T20:21:24,521 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:24,521 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:24,521 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. after waiting 0 ms 2024-11-07T20:21:24,521 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:24,521 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:24,521 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 80eec78f120d1efdd45815dfc102e429: 2024-11-07T20:21:24,522 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:21:24,522 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731010884522"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010884522"}]},"ts":"1731010884522"} 2024-11-07T20:21:24,524 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:21:24,525 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:21:24,525 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010884525"}]},"ts":"1731010884525"} 2024-11-07T20:21:24,526 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:21:24,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=80eec78f120d1efdd45815dfc102e429, ASSIGN}] 2024-11-07T20:21:24,530 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=80eec78f120d1efdd45815dfc102e429, ASSIGN 2024-11-07T20:21:24,530 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=80eec78f120d1efdd45815dfc102e429, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:21:24,681 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=80eec78f120d1efdd45815dfc102e429, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:24,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure 80eec78f120d1efdd45815dfc102e429, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:21:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T20:21:24,834 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:21:24,840 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:24,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => 80eec78f120d1efdd45815dfc102e429, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:21:24,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true 80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:24,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:24,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for 80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:24,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for 80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:24,842 INFO [StoreOpener-80eec78f120d1efdd45815dfc102e429-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:24,843 INFO [StoreOpener-80eec78f120d1efdd45815dfc102e429-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80eec78f120d1efdd45815dfc102e429 columnFamilyName testFamily 2024-11-07T20:21:24,843 DEBUG [StoreOpener-80eec78f120d1efdd45815dfc102e429-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:24,844 INFO [StoreOpener-80eec78f120d1efdd45815dfc102e429-1 {}] regionserver.HStore(327): Store=80eec78f120d1efdd45815dfc102e429/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:21:24,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:24,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:24,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for 80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:24,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:21:24,850 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened 80eec78f120d1efdd45815dfc102e429; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61432501, jitterRate=-0.08458440005779266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:21:24,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for 80eec78f120d1efdd45815dfc102e429: 2024-11-07T20:21:24,851 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429., pid=89, masterSystemTime=1731010884834 2024-11-07T20:21:24,852 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:24,852 INFO [RS_OPEN_PRIORITY_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:24,853 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=80eec78f120d1efdd45815dfc102e429, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:24,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T20:21:24,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure 80eec78f120d1efdd45815dfc102e429, server=15c0e1002410,32779,1731010677532 in 171 msec 2024-11-07T20:21:24,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-07T20:21:24,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=80eec78f120d1efdd45815dfc102e429, ASSIGN in 327 msec 2024-11-07T20:21:24,857 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:21:24,857 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010884857"}]},"ts":"1731010884857"} 2024-11-07T20:21:24,859 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:21:24,861 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:21:24,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1600 sec 2024-11-07T20:21:25,282 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:21:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-07T20:21:26,706 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-07T20:21:26,707 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:21:26,707 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:26,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-07T20:21:26,710 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645920 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:21:26,712 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-07T20:21:26,712 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:26,712 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-07T20:21:26,714 DEBUG [hconnection-0x79824090-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:21:26,715 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54740, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:21:26,721 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:21:26,723 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-07T20:21:26,724 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-07T20:21:26,724 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645923 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:21:26,725 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-07T20:21:26,726 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645923 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:21:26,726 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-07T20:21:27,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-07T20:21:27,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:21:27,471 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905746 expired clientIPAndPort=172.17.0.2:34650, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:21:28,071 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905739 expired clientIPAndPort=172.17.0.2:36922, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:28,071 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905742 expired clientIPAndPort=172.17.0.2:36922, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:28,072 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905740 expired clientIPAndPort=172.17.0.2:36922, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:28,190 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:21:30,374 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:21:30,413 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-07T20:21:31,174 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905733 expired clientIPAndPort=172.17.0.2:36936, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:21:35,729 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429., hostname=15c0e1002410,32779,1731010677532, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429., hostname=15c0e1002410,32779,1731010677532, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9002ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-07T20:21:35,729 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9004 2024-11-07T20:21:35,729 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x760de6d1 to 127.0.0.1:59122 2024-11-07T20:21:35,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:35,729 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:21:35,730 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:21:35,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:35,730 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d9612bd to 127.0.0.1:59122 2024-11-07T20:21:35,739 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=227 (was 228), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=5757 (was 5848) 2024-11-07T20:21:35,747 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=5757 2024-11-07T20:21:35,748 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bf05785 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48ceebd8 2024-11-07T20:21:35,751 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6bf05785 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5201dc2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:35,751 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2ca77db3 to 127.0.0.1:59122 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c2b1fc6 2024-11-07T20:21:35,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458e29b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T20:21:35,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T20:21:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T20:21:35,760 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T20:21:35,760 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:35,760 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-07T20:21:35,761 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T20:21:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T20:21:35,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741891_1067 (size=431) 2024-11-07T20:21:35,777 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905728 expired clientIPAndPort=172.17.0.2:58750, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:21:36,173 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 7eb2b0e0b365f15a0f88a3976d4e4d5d, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7 2024-11-07T20:21:36,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741892_1068 (size=82) 2024-11-07T20:21:36,226 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:21:36,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:36,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 7eb2b0e0b365f15a0f88a3976d4e4d5d, disabling compactions & flushes 2024-11-07T20:21:36,580 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:36,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:36,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. after waiting 0 ms 2024-11-07T20:21:36,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:36,580 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:36,580 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 7eb2b0e0b365f15a0f88a3976d4e4d5d: 2024-11-07T20:21:36,582 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T20:21:36,582 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731010896582"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731010896582"}]},"ts":"1731010896582"} 2024-11-07T20:21:36,583 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-07T20:21:36,584 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T20:21:36,584 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010896584"}]},"ts":"1731010896584"} 2024-11-07T20:21:36,585 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-07T20:21:36,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=7eb2b0e0b365f15a0f88a3976d4e4d5d, ASSIGN}] 2024-11-07T20:21:36,589 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=7eb2b0e0b365f15a0f88a3976d4e4d5d, ASSIGN 2024-11-07T20:21:36,590 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=7eb2b0e0b365f15a0f88a3976d4e4d5d, ASSIGN; state=OFFLINE, location=15c0e1002410,32779,1731010677532; forceNewPlan=false, retain=false 2024-11-07T20:21:36,740 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=7eb2b0e0b365f15a0f88a3976d4e4d5d, regionState=OPENING, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:36,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 7eb2b0e0b365f15a0f88a3976d4e4d5d, server=15c0e1002410,32779,1731010677532}] 2024-11-07T20:21:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T20:21:36,894 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 15c0e1002410,32779,1731010677532 2024-11-07T20:21:36,899 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:36,900 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 7eb2b0e0b365f15a0f88a3976d4e4d5d, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d.', STARTKEY => '', ENDKEY => ''} 2024-11-07T20:21:36,900 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:36,900 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T20:21:36,900 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:36,900 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:36,902 INFO [StoreOpener-7eb2b0e0b365f15a0f88a3976d4e4d5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:36,903 INFO [StoreOpener-7eb2b0e0b365f15a0f88a3976d4e4d5d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eb2b0e0b365f15a0f88a3976d4e4d5d columnFamilyName testFamily 2024-11-07T20:21:36,903 DEBUG [StoreOpener-7eb2b0e0b365f15a0f88a3976d4e4d5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T20:21:36,904 INFO [StoreOpener-7eb2b0e0b365f15a0f88a3976d4e4d5d-1 {}] regionserver.HStore(327): Store=7eb2b0e0b365f15a0f88a3976d4e4d5d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T20:21:36,904 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:36,905 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:36,907 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:36,909 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T20:21:36,909 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 7eb2b0e0b365f15a0f88a3976d4e4d5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71885587, jitterRate=0.07117871940135956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T20:21:36,910 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 7eb2b0e0b365f15a0f88a3976d4e4d5d: 2024-11-07T20:21:36,911 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., pid=92, masterSystemTime=1731010896894 2024-11-07T20:21:36,912 DEBUG [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:36,912 INFO [RS_OPEN_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:36,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=7eb2b0e0b365f15a0f88a3976d4e4d5d, regionState=OPEN, openSeqNum=2, regionLocation=15c0e1002410,32779,1731010677532 2024-11-07T20:21:36,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T20:21:36,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 7eb2b0e0b365f15a0f88a3976d4e4d5d, server=15c0e1002410,32779,1731010677532 in 171 msec 2024-11-07T20:21:36,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-07T20:21:36,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=7eb2b0e0b365f15a0f88a3976d4e4d5d, ASSIGN in 327 msec 2024-11-07T20:21:36,917 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T20:21:36,917 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731010896917"}]},"ts":"1731010896917"} 2024-11-07T20:21:36,918 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-07T20:21:36,921 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T20:21:36,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1630 sec 2024-11-07T20:21:37,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T20:21:37,041 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-07T20:21:37,779 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905726 expired clientIPAndPort=172.17.0.2:58750, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:21:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46471 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-07T20:21:38,764 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-07T20:21:38,764 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-07T20:21:38,765 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:38,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32779 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-07T20:21:38,769 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 12029644771174645924 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-07T20:21:38,770 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-07T20:21:38,770 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T20:21:38,770 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-07T20:21:38,772 DEBUG [hconnection-0x5baa4dd0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:21:38,773 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50076, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:21:38,780 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-07T20:21:38,782 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-07T20:21:38,782 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-07T20:21:38,783 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6bf05785 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=15c0e1002410,32779,1731010677532, seqNum=-1] 2024-11-07T20:21:38,783 DEBUG [ReadOnlyZKClient-127.0.0.1:59122@0x6bf05785 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T20:21:38,784 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50086, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T20:21:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:21:38,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:21:39,788 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-07T20:21:39,789 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-07T20:21:39,789 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:21:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-07T20:21:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2] 2024-11-07T20:21:40,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32779 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-07T20:21:41,644 ERROR [LeaseRenewer:jenkins@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:39675,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:21:41,788 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=983ms 2024-11-07T20:21:41,789 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=983ms 2024-11-07T20:21:41,789 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d., hostname=15c0e1002410,32779,1731010677532, seqNum=2 from cache 2024-11-07T20:21:41,789 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-07T20:21:39.789Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-07T20:21:41.789Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=983ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=15c0e1002410:32779 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-07T20:21:41,790 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3008 2024-11-07T20:21:41,790 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2ca77db3 to 127.0.0.1:59122 2024-11-07T20:21:41,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:41,790 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-07T20:21:41,790 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T20:21:41,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:41,790 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6bf05785 to 127.0.0.1:59122 2024-11-07T20:21:41,799 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=230 (was 227) - Thread LEAK? -, OpenFileDescriptor=449 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=5748 (was 5757) 2024-11-07T20:21:41,800 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-07T20:21:41,800 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-07T20:21:41,800 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x551b3f6b to 127.0.0.1:59122 2024-11-07T20:21:41,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:41,800 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T20:21:41,800 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2091041397, stopped=false 2024-11-07T20:21:41,800 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=15c0e1002410,46471,1731010676369 2024-11-07T20:21:41,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T20:21:41,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T20:21:41,802 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-07T20:21:41,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:21:41,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:21:41,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:41,803 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '15c0e1002410,32779,1731010677532' ***** 2024-11-07T20:21:41,803 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-07T20:21:41,803 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T20:21:41,803 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T20:21:41,803 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T20:21:41,804 INFO [RS:0;15c0e1002410:32779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T20:21:41,804 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-07T20:21:41,804 INFO [RS:0;15c0e1002410:32779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T20:21:41,804 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for e0438524b59c2ac6ff13273e4babab72 2024-11-07T20:21:41,804 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 3a0e4a19c977ab180d4a99c8159e7bfa 2024-11-07T20:21:41,804 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 86e819bb0a1f1ab205ac418e88f4cc7b 2024-11-07T20:21:41,804 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 7eb2b0e0b365f15a0f88a3976d4e4d5d 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 61a760d6216fe8653467451e266dc60e 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 2416e7f18037c665ca8a4b6f6d200c32 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 5506bb10abae8b4dfe00662cb6db8777 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 2f3f6d4a4f2ed609a76a060c63de295e 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for ab0c24b781f1e9f98f3a812c0ae403d8 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for b65a37ef0d404205cc6dcabe718de758 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 4ae63f3930be22567e3de62358f605f4 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for eb505c3486a2bcb859f8a689ce4a1422 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for b7c9dd8db5786caaf3eace20a2133768 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 7a14658215f07a3fd882c2a428ad1d0d 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 7ead674e6838d105f6f8e11d652b860c 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 5dbcc8e35a32fe336d087243233e5f12 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 80eec78f120d1efdd45815dfc102e429 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for dd525a3c56e204c238b5565ab48afa57 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for c3c1af47e03021f6356fe5780cb33e13 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 2683d6dd943cbb13e85b92ced21cdb05 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for acbc84b2cbff5d121607335cd7f60cb9 2024-11-07T20:21:41,805 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e0438524b59c2ac6ff13273e4babab72, disabling compactions & flushes 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 45f0f6007cab85b2d5717f89f96bb49b 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for aa9aeb512dbdd6d51b6c45aacf3be68c 2024-11-07T20:21:41,805 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for c20367372a9444f76086b80bb0cd5335 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for d1553686ddc24d1772f86227f55ddbce 2024-11-07T20:21:41,805 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:41,805 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. after waiting 0 ms 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1224): stopping server 15c0e1002410,32779,1731010677532 2024-11-07T20:21:41,805 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:21:41,805 DEBUG [RS:0;15c0e1002410:32779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T20:21:41,805 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-07T20:21:41,806 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-07T20:21:41,806 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e0438524b59c2ac6ff13273e4babab72 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:41,806 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-07T20:21:41,806 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-07T20:21:41,806 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-07T20:21:41,806 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T20:21:41,806 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T20:21:41,806 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-07T20:21:41,806 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 3a0e4a19c977ab180d4a99c8159e7bfa, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, 86e819bb0a1f1ab205ac418e88f4cc7b, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e0438524b59c2ac6ff13273e4babab72, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:41,861 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72/.tmp/testFamily/2f0bd7ed5ca24f9b858d246c8a1858d0 is 61, key is row-0/testFamily:testQualifier/1731010829136/Put/seqid=0 2024-11-07T20:21:41,869 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/.tmp/info/74612eb63d8249a2acd8b611ff33eac8 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57./info:regioninfo/1731010713377/Put/seqid=0 2024-11-07T20:21:41,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741893_1069 (size=5049) 2024-11-07T20:21:41,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741894_1070 (size=51873) 2024-11-07T20:21:41,882 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -6417099302534905721 expired clientIPAndPort=172.17.0.2:58772, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:21:41,907 INFO [regionserver/15c0e1002410:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T20:21:41,907 INFO [regionserver/15c0e1002410:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T20:21:42,006 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 3a0e4a19c977ab180d4a99c8159e7bfa, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, 86e819bb0a1f1ab205ac418e88f4cc7b, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e0438524b59c2ac6ff13273e4babab72, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:42,207 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 3a0e4a19c977ab180d4a99c8159e7bfa, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, 86e819bb0a1f1ab205ac418e88f4cc7b, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e0438524b59c2ac6ff13273e4babab72, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:42,271 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72/.tmp/testFamily/2f0bd7ed5ca24f9b858d246c8a1858d0 2024-11-07T20:21:42,275 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/.tmp/info/74612eb63d8249a2acd8b611ff33eac8 2024-11-07T20:21:42,300 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-07T20:21:42,316 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72/.tmp/testFamily/2f0bd7ed5ca24f9b858d246c8a1858d0 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72/testFamily/2f0bd7ed5ca24f9b858d246c8a1858d0 2024-11-07T20:21:42,324 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72/testFamily/2f0bd7ed5ca24f9b858d246c8a1858d0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:42,327 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e0438524b59c2ac6ff13273e4babab72 in 520ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:42,332 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_1_-true/e0438524b59c2ac6ff13273e4babab72/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:42,334 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:21:42,334 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e0438524b59c2ac6ff13273e4babab72: 2024-11-07T20:21:42,334 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731010828115.e0438524b59c2ac6ff13273e4babab72. 2024-11-07T20:21:42,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3a0e4a19c977ab180d4a99c8159e7bfa, disabling compactions & flushes 2024-11-07T20:21:42,335 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:21:42,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:21:42,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. after waiting 0 ms 2024-11-07T20:21:42,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:21:42,335 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3a0e4a19c977ab180d4a99c8159e7bfa 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:42,335 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/.tmp/table/887a2501195e48ecb3c56ca4cdf061dc is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1731010713389/Put/seqid=0 2024-11-07T20:21:42,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741895_1071 (size=9821) 2024-11-07T20:21:42,350 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa/.tmp/testFamily/8b8fdbe8da2847c4993220b47edc84f2 is 61, key is row-0/testFamily:testQualifier/1731010753152/Put/seqid=0 2024-11-07T20:21:42,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741896_1072 (size=5049) 2024-11-07T20:21:42,355 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa/.tmp/testFamily/8b8fdbe8da2847c4993220b47edc84f2 2024-11-07T20:21:42,361 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa/.tmp/testFamily/8b8fdbe8da2847c4993220b47edc84f2 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa/testFamily/8b8fdbe8da2847c4993220b47edc84f2 2024-11-07T20:21:42,368 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa/testFamily/8b8fdbe8da2847c4993220b47edc84f2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:42,369 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3a0e4a19c977ab180d4a99c8159e7bfa in 34ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:42,373 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_0_-false/3a0e4a19c977ab180d4a99c8159e7bfa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:42,373 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:21:42,373 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3a0e4a19c977ab180d4a99c8159e7bfa: 2024-11-07T20:21:42,373 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731010752128.3a0e4a19c977ab180d4a99c8159e7bfa. 2024-11-07T20:21:42,374 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 86e819bb0a1f1ab205ac418e88f4cc7b, disabling compactions & flushes 2024-11-07T20:21:42,374 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:21:42,374 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:21:42,374 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. after waiting 0 ms 2024-11-07T20:21:42,374 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:21:42,374 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 86e819bb0a1f1ab205ac418e88f4cc7b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:42,389 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b/.tmp/testFamily/653a5a49f6f841e38575b3929730dcd2 is 61, key is row-0/testFamily:testQualifier/1731010844356/Put/seqid=0 2024-11-07T20:21:42,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741897_1073 (size=5049) 2024-11-07T20:21:42,407 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, 86e819bb0a1f1ab205ac418e88f4cc7b, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:42,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:21:42,475 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-07T20:21:42,607 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, 86e819bb0a1f1ab205ac418e88f4cc7b, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:42,741 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/.tmp/table/887a2501195e48ecb3c56ca4cdf061dc 2024-11-07T20:21:42,748 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/.tmp/info/74612eb63d8249a2acd8b611ff33eac8 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/info/74612eb63d8249a2acd8b611ff33eac8 2024-11-07T20:21:42,755 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/info/74612eb63d8249a2acd8b611ff33eac8, entries=290, sequenceid=149, filesize=50.7 K 2024-11-07T20:21:42,756 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/.tmp/table/887a2501195e48ecb3c56ca4cdf061dc as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/table/887a2501195e48ecb3c56ca4cdf061dc 2024-11-07T20:21:42,762 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/table/887a2501195e48ecb3c56ca4cdf061dc, entries=58, sequenceid=149, filesize=9.6 K 2024-11-07T20:21:42,763 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 957ms, sequenceid=149, compaction requested=false 2024-11-07T20:21:42,767 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-07T20:21:42,768 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T20:21:42,768 INFO [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-07T20:21:42,768 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-07T20:21:42,768 DEBUG [RS_CLOSE_META-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T20:21:42,793 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:39675 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:39675,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T20:21:42,794 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b/.tmp/testFamily/653a5a49f6f841e38575b3929730dcd2 2024-11-07T20:21:42,801 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b/.tmp/testFamily/653a5a49f6f841e38575b3929730dcd2 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b/testFamily/653a5a49f6f841e38575b3929730dcd2 2024-11-07T20:21:42,807 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-07T20:21:42,808 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, 86e819bb0a1f1ab205ac418e88f4cc7b, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:42,808 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b/testFamily/653a5a49f6f841e38575b3929730dcd2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:42,809 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 86e819bb0a1f1ab205ac418e88f4cc7b in 435ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:42,813 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnClose_1_-true/86e819bb0a1f1ab205ac418e88f4cc7b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:42,814 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:21:42,814 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 86e819bb0a1f1ab205ac418e88f4cc7b: 2024-11-07T20:21:42,814 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731010841329.86e819bb0a1f1ab205ac418e88f4cc7b. 2024-11-07T20:21:42,814 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7eb2b0e0b365f15a0f88a3976d4e4d5d, disabling compactions & flushes 2024-11-07T20:21:42,814 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:42,814 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:42,814 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. after waiting 0 ms 2024-11-07T20:21:42,814 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:42,815 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7eb2b0e0b365f15a0f88a3976d4e4d5d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:42,830 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d/.tmp/testFamily/0e01bc26648c406c85ea933366a4ff5a is 61, key is row-0/testFamily:testQualifier/1731010898775/Put/seqid=0 2024-11-07T20:21:42,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741898_1074 (size=5049) 2024-11-07T20:21:43,008 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:43,208 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 7eb2b0e0b365f15a0f88a3976d4e4d5d, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:43,235 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d/.tmp/testFamily/0e01bc26648c406c85ea933366a4ff5a 2024-11-07T20:21:43,243 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d/.tmp/testFamily/0e01bc26648c406c85ea933366a4ff5a as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d/testFamily/0e01bc26648c406c85ea933366a4ff5a 2024-11-07T20:21:43,250 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d/testFamily/0e01bc26648c406c85ea933366a4ff5a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:43,253 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7eb2b0e0b365f15a0f88a3976d4e4d5d in 438ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:43,257 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/7eb2b0e0b365f15a0f88a3976d4e4d5d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:43,258 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:43,258 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7eb2b0e0b365f15a0f88a3976d4e4d5d: 2024-11-07T20:21:43,258 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731010895758.7eb2b0e0b365f15a0f88a3976d4e4d5d. 2024-11-07T20:21:43,258 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 61a760d6216fe8653467451e266dc60e, disabling compactions & flushes 2024-11-07T20:21:43,258 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:21:43,258 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:21:43,258 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. after waiting 0 ms 2024-11-07T20:21:43,258 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:21:43,258 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 61a760d6216fe8653467451e266dc60e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:43,274 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e/.tmp/testFamily/dce22fdebe0d40ba9c282e38705dba69 is 61, key is row-0/testFamily:testQualifier/1731010738676/Put/seqid=0 2024-11-07T20:21:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741899_1075 (size=5049) 2024-11-07T20:21:43,408 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:43,609 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 61a760d6216fe8653467451e266dc60e, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:43,679 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e/.tmp/testFamily/dce22fdebe0d40ba9c282e38705dba69 2024-11-07T20:21:43,686 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e/.tmp/testFamily/dce22fdebe0d40ba9c282e38705dba69 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e/testFamily/dce22fdebe0d40ba9c282e38705dba69 2024-11-07T20:21:43,692 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e/testFamily/dce22fdebe0d40ba9c282e38705dba69, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:43,693 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 61a760d6216fe8653467451e266dc60e in 435ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:43,697 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_0_-false/61a760d6216fe8653467451e266dc60e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:43,698 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:21:43,698 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 61a760d6216fe8653467451e266dc60e: 2024-11-07T20:21:43,698 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731010737620.61a760d6216fe8653467451e266dc60e. 2024-11-07T20:21:43,698 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2416e7f18037c665ca8a4b6f6d200c32, disabling compactions & flushes 2024-11-07T20:21:43,698 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:43,698 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:43,698 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. after waiting 0 ms 2024-11-07T20:21:43,698 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:43,698 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2416e7f18037c665ca8a4b6f6d200c32 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:43,714 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32/.tmp/testFamily/0810f298a5eb45ed8f5780c39d5a0076 is 61, key is row-0/testFamily:testQualifier/1731010828018/Put/seqid=0 2024-11-07T20:21:43,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741900_1076 (size=5049) 2024-11-07T20:21:43,809 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 24 regions to close 2024-11-07T20:21:43,809 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:44,009 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2416e7f18037c665ca8a4b6f6d200c32, 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:44,119 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32/.tmp/testFamily/0810f298a5eb45ed8f5780c39d5a0076 2024-11-07T20:21:44,126 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32/.tmp/testFamily/0810f298a5eb45ed8f5780c39d5a0076 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32/testFamily/0810f298a5eb45ed8f5780c39d5a0076 2024-11-07T20:21:44,132 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32/testFamily/0810f298a5eb45ed8f5780c39d5a0076, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:44,133 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2416e7f18037c665ca8a4b6f6d200c32 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:44,136 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextException_1_-true/2416e7f18037c665ca8a4b6f6d200c32/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:44,137 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:44,137 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2416e7f18037c665ca8a4b6f6d200c32: 2024-11-07T20:21:44,137 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731010826994.2416e7f18037c665ca8a4b6f6d200c32. 2024-11-07T20:21:44,137 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5506bb10abae8b4dfe00662cb6db8777, disabling compactions & flushes 2024-11-07T20:21:44,137 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:21:44,137 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:21:44,137 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. after waiting 0 ms 2024-11-07T20:21:44,137 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:21:44,137 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 5506bb10abae8b4dfe00662cb6db8777 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:44,153 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777/.tmp/testFamily/29e3fcb4d4bf461599c9b929977caae9 is 61, key is row-0/testFamily:testQualifier/1731010739840/Put/seqid=0 2024-11-07T20:21:44,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741901_1077 (size=5049) 2024-11-07T20:21:44,210 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:44,410 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5506bb10abae8b4dfe00662cb6db8777, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:44,558 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777/.tmp/testFamily/29e3fcb4d4bf461599c9b929977caae9 2024-11-07T20:21:44,565 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777/.tmp/testFamily/29e3fcb4d4bf461599c9b929977caae9 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777/testFamily/29e3fcb4d4bf461599c9b929977caae9 2024-11-07T20:21:44,571 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777/testFamily/29e3fcb4d4bf461599c9b929977caae9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:44,572 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5506bb10abae8b4dfe00662cb6db8777 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:44,576 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnRenewLease_0_-false/5506bb10abae8b4dfe00662cb6db8777/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:44,577 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:21:44,577 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5506bb10abae8b4dfe00662cb6db8777: 2024-11-07T20:21:44,577 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731010738791.5506bb10abae8b4dfe00662cb6db8777. 2024-11-07T20:21:44,577 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2f3f6d4a4f2ed609a76a060c63de295e, disabling compactions & flushes 2024-11-07T20:21:44,577 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:21:44,577 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:21:44,577 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. after waiting 0 ms 2024-11-07T20:21:44,577 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:21:44,577 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2f3f6d4a4f2ed609a76a060c63de295e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:44,593 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e/.tmp/testFamily/b0b43f3ea8c0414cbbf6bc7c540b85ee is 61, key is row-0/testFamily:testQualifier/1731010763335/Put/seqid=0 2024-11-07T20:21:44,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741902_1078 (size=5049) 2024-11-07T20:21:44,610 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:44,810 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 22 regions to close 2024-11-07T20:21:44,810 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:44,998 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e/.tmp/testFamily/b0b43f3ea8c0414cbbf6bc7c540b85ee 2024-11-07T20:21:45,006 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e/.tmp/testFamily/b0b43f3ea8c0414cbbf6bc7c540b85ee as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e/testFamily/b0b43f3ea8c0414cbbf6bc7c540b85ee 2024-11-07T20:21:45,011 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 2f3f6d4a4f2ed609a76a060c63de295e, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:45,012 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e/testFamily/b0b43f3ea8c0414cbbf6bc7c540b85ee, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:45,013 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2f3f6d4a4f2ed609a76a060c63de295e in 436ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:45,017 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_0_-false/2f3f6d4a4f2ed609a76a060c63de295e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:45,018 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:21:45,018 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2f3f6d4a4f2ed609a76a060c63de295e: 2024-11-07T20:21:45,018 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1731010762314.2f3f6d4a4f2ed609a76a060c63de295e. 2024-11-07T20:21:45,018 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f49676cac4562adaefc259527bfa03c5, disabling compactions & flushes 2024-11-07T20:21:45,018 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:21:45,018 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:21:45,018 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. after waiting 0 ms 2024-11-07T20:21:45,018 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:21:45,018 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f49676cac4562adaefc259527bfa03c5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:45,034 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5/.tmp/testFamily/42026a8d96f44c978db986da8368d8f9 is 61, key is row-0/testFamily:testQualifier/1731010858488/Put/seqid=0 2024-11-07T20:21:45,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741903_1079 (size=5049) 2024-11-07T20:21:45,211 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:45,411 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa, f49676cac4562adaefc259527bfa03c5 2024-11-07T20:21:45,439 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5/.tmp/testFamily/42026a8d96f44c978db986da8368d8f9 2024-11-07T20:21:45,446 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5/.tmp/testFamily/42026a8d96f44c978db986da8368d8f9 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5/testFamily/42026a8d96f44c978db986da8368d8f9 2024-11-07T20:21:45,452 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5/testFamily/42026a8d96f44c978db986da8368d8f9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:45,453 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f49676cac4562adaefc259527bfa03c5 in 435ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:45,457 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNext_1_-true/f49676cac4562adaefc259527bfa03c5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:45,458 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:21:45,458 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f49676cac4562adaefc259527bfa03c5: 2024-11-07T20:21:45,458 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731010855472.f49676cac4562adaefc259527bfa03c5. 2024-11-07T20:21:45,458 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ab0c24b781f1e9f98f3a812c0ae403d8, disabling compactions & flushes 2024-11-07T20:21:45,458 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:21:45,458 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:21:45,458 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. after waiting 0 ms 2024-11-07T20:21:45,458 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:21:45,458 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ab0c24b781f1e9f98f3a812c0ae403d8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:45,473 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8/.tmp/testFamily/acdd9014f2df40bc81bcf1866f6d8488 is 61, key is row-0/testFamily:testQualifier/1731010777554/Put/seqid=0 2024-11-07T20:21:45,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741904_1080 (size=5049) 2024-11-07T20:21:45,611 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:45,812 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 20 regions to close 2024-11-07T20:21:45,812 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, ab0c24b781f1e9f98f3a812c0ae403d8, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:45,878 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8/.tmp/testFamily/acdd9014f2df40bc81bcf1866f6d8488 2024-11-07T20:21:45,885 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8/.tmp/testFamily/acdd9014f2df40bc81bcf1866f6d8488 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8/testFamily/acdd9014f2df40bc81bcf1866f6d8488 2024-11-07T20:21:45,891 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8/testFamily/acdd9014f2df40bc81bcf1866f6d8488, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:45,892 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ab0c24b781f1e9f98f3a812c0ae403d8 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:45,896 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/ab0c24b781f1e9f98f3a812c0ae403d8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:45,896 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:21:45,896 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ab0c24b781f1e9f98f3a812c0ae403d8: 2024-11-07T20:21:45,896 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731010774525.ab0c24b781f1e9f98f3a812c0ae403d8. 2024-11-07T20:21:45,896 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b65a37ef0d404205cc6dcabe718de758, disabling compactions & flushes 2024-11-07T20:21:45,896 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:45,896 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:45,896 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. after waiting 0 ms 2024-11-07T20:21:45,897 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:45,897 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b65a37ef0d404205cc6dcabe718de758 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:45,912 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758/.tmp/testFamily/0a02d8527ec04a43a065ccc0cf19517c is 61, key is row-0/testFamily:testQualifier/1731010874653/Put/seqid=0 2024-11-07T20:21:45,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741905_1081 (size=5049) 2024-11-07T20:21:46,012 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:46,212 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b65a37ef0d404205cc6dcabe718de758, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:46,316 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758/.tmp/testFamily/0a02d8527ec04a43a065ccc0cf19517c 2024-11-07T20:21:46,324 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758/.tmp/testFamily/0a02d8527ec04a43a065ccc0cf19517c as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758/testFamily/0a02d8527ec04a43a065ccc0cf19517c 2024-11-07T20:21:46,329 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758/testFamily/0a02d8527ec04a43a065ccc0cf19517c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:46,330 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b65a37ef0d404205cc6dcabe718de758 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:46,334 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/b65a37ef0d404205cc6dcabe718de758/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:46,335 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:46,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b65a37ef0d404205cc6dcabe718de758: 2024-11-07T20:21:46,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731010871636.b65a37ef0d404205cc6dcabe718de758. 2024-11-07T20:21:46,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4ae63f3930be22567e3de62358f605f4, disabling compactions & flushes 2024-11-07T20:21:46,335 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:21:46,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:21:46,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. after waiting 0 ms 2024-11-07T20:21:46,335 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:21:46,335 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4ae63f3930be22567e3de62358f605f4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:46,350 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4/.tmp/testFamily/a7238a02f828468eb9dcdf45ea3134ae is 61, key is row-0/testFamily:testQualifier/1731010728460/Put/seqid=0 2024-11-07T20:21:46,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741906_1082 (size=5049) 2024-11-07T20:21:46,412 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:46,613 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 4ae63f3930be22567e3de62358f605f4, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:46,755 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4/.tmp/testFamily/a7238a02f828468eb9dcdf45ea3134ae 2024-11-07T20:21:46,762 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4/.tmp/testFamily/a7238a02f828468eb9dcdf45ea3134ae as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4/testFamily/a7238a02f828468eb9dcdf45ea3134ae 2024-11-07T20:21:46,768 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4/testFamily/a7238a02f828468eb9dcdf45ea3134ae, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:46,769 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4ae63f3930be22567e3de62358f605f4 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:46,772 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/4ae63f3930be22567e3de62358f605f4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:46,773 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:21:46,773 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4ae63f3930be22567e3de62358f605f4: 2024-11-07T20:21:46,773 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731010727434.4ae63f3930be22567e3de62358f605f4. 2024-11-07T20:21:46,773 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing eb505c3486a2bcb859f8a689ce4a1422, disabling compactions & flushes 2024-11-07T20:21:46,773 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:46,773 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:46,773 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. after waiting 0 ms 2024-11-07T20:21:46,773 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:46,773 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing eb505c3486a2bcb859f8a689ce4a1422 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:46,792 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422/.tmp/testFamily/73a64f44564d4ae98862754f160a3838 is 61, key is row-0/testFamily:testQualifier/1731010864557/Put/seqid=0 2024-11-07T20:21:46,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741907_1083 (size=5049) 2024-11-07T20:21:46,813 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 17 regions to close 2024-11-07T20:21:46,813 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:47,013 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T20:21:47,041 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-07T20:21:47,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T20:21:47,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-07T20:21:47,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-07T20:21:47,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-07T20:21:47,197 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422/.tmp/testFamily/73a64f44564d4ae98862754f160a3838 2024-11-07T20:21:47,204 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422/.tmp/testFamily/73a64f44564d4ae98862754f160a3838 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422/testFamily/73a64f44564d4ae98862754f160a3838 2024-11-07T20:21:47,210 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422/testFamily/73a64f44564d4ae98862754f160a3838, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:47,211 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb505c3486a2bcb859f8a689ce4a1422 in 437ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:47,213 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eb505c3486a2bcb859f8a689ce4a1422, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:47,215 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/eb505c3486a2bcb859f8a689ce4a1422/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:47,215 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:47,215 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for eb505c3486a2bcb859f8a689ce4a1422: 2024-11-07T20:21:47,215 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731010861539.eb505c3486a2bcb859f8a689ce4a1422. 2024-11-07T20:21:47,215 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing b7c9dd8db5786caaf3eace20a2133768, disabling compactions & flushes 2024-11-07T20:21:47,215 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:21:47,215 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:21:47,215 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. after waiting 0 ms 2024-11-07T20:21:47,215 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:21:47,216 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing b7c9dd8db5786caaf3eace20a2133768 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:47,231 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768/.tmp/testFamily/9ff2da4f0fbe45cbbb23b1b20387e440 is 61, key is row-0/testFamily:testQualifier/1731010817926/Put/seqid=0 2024-11-07T20:21:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741908_1084 (size=5049) 2024-11-07T20:21:47,414 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:47,614 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, b7c9dd8db5786caaf3eace20a2133768, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:47,636 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768/.tmp/testFamily/9ff2da4f0fbe45cbbb23b1b20387e440 2024-11-07T20:21:47,643 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768/.tmp/testFamily/9ff2da4f0fbe45cbbb23b1b20387e440 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768/testFamily/9ff2da4f0fbe45cbbb23b1b20387e440 2024-11-07T20:21:47,649 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768/testFamily/9ff2da4f0fbe45cbbb23b1b20387e440, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:47,650 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b7c9dd8db5786caaf3eace20a2133768 in 434ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:47,654 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/b7c9dd8db5786caaf3eace20a2133768/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:47,655 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:21:47,655 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for b7c9dd8db5786caaf3eace20a2133768: 2024-11-07T20:21:47,655 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731010816905.b7c9dd8db5786caaf3eace20a2133768. 2024-11-07T20:21:47,655 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7a14658215f07a3fd882c2a428ad1d0d, disabling compactions & flushes 2024-11-07T20:21:47,655 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:21:47,655 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:21:47,655 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. after waiting 0 ms 2024-11-07T20:21:47,655 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:21:47,656 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7a14658215f07a3fd882c2a428ad1d0d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:47,671 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d/.tmp/testFamily/9d3d7b009a424ca0814b0662eace3446 is 61, key is row-0/testFamily:testQualifier/1731010853928/Put/seqid=0 2024-11-07T20:21:47,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741909_1085 (size=5049) 2024-11-07T20:21:47,688 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(140): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=15c0e1002410:46471 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.getLiveRegionServers(RegionServerStatusProtos.java:17362) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionImplementation.getLiveRegionServers(ConnectionImplementation.java:2376) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:138) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 12 more 2024-11-07T20:21:47,814 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 15 regions to close 2024-11-07T20:21:47,814 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:47,836 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:21:48,014 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7a14658215f07a3fd882c2a428ad1d0d, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:48,076 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d/.tmp/testFamily/9d3d7b009a424ca0814b0662eace3446 2024-11-07T20:21:48,083 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d/.tmp/testFamily/9d3d7b009a424ca0814b0662eace3446 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d/testFamily/9d3d7b009a424ca0814b0662eace3446 2024-11-07T20:21:48,089 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d/testFamily/9d3d7b009a424ca0814b0662eace3446, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:48,090 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7a14658215f07a3fd882c2a428ad1d0d in 434ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:48,094 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_1_-true/7a14658215f07a3fd882c2a428ad1d0d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:48,094 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:21:48,094 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7a14658215f07a3fd882c2a428ad1d0d: 2024-11-07T20:21:48,094 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731010850904.7a14658215f07a3fd882c2a428ad1d0d. 2024-11-07T20:21:48,094 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 7ead674e6838d105f6f8e11d652b860c, disabling compactions & flushes 2024-11-07T20:21:48,094 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:21:48,095 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:21:48,095 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. after waiting 0 ms 2024-11-07T20:21:48,095 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:21:48,095 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 7ead674e6838d105f6f8e11d652b860c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:48,110 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c/.tmp/testFamily/2b17a5d94528489db110843095039a92 is 61, key is row-0/testFamily:testQualifier/1731010767454/Put/seqid=0 2024-11-07T20:21:48,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741910_1086 (size=5049) 2024-11-07T20:21:48,215 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:48,415 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 7ead674e6838d105f6f8e11d652b860c, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:48,515 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c/.tmp/testFamily/2b17a5d94528489db110843095039a92 2024-11-07T20:21:48,522 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c/.tmp/testFamily/2b17a5d94528489db110843095039a92 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c/testFamily/2b17a5d94528489db110843095039a92 2024-11-07T20:21:48,527 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c/testFamily/2b17a5d94528489db110843095039a92, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:48,528 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ead674e6838d105f6f8e11d652b860c in 433ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:48,532 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/7ead674e6838d105f6f8e11d652b860c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:48,533 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:21:48,533 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 7ead674e6838d105f6f8e11d652b860c: 2024-11-07T20:21:48,533 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731010764427.7ead674e6838d105f6f8e11d652b860c. 2024-11-07T20:21:48,533 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing eba09a860a3a6c62a1b5d86497c767fa, disabling compactions & flushes 2024-11-07T20:21:48,533 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:21:48,533 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:21:48,533 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. after waiting 0 ms 2024-11-07T20:21:48,533 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:21:48,533 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing eba09a860a3a6c62a1b5d86497c767fa 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:48,548 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa/.tmp/testFamily/528859907a1f4e07882a9ecc2656c1a5 is 61, key is row-0/testFamily:testQualifier/1731010804766/Put/seqid=0 2024-11-07T20:21:48,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741911_1087 (size=5049) 2024-11-07T20:21:48,615 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:48,815 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 13 regions to close 2024-11-07T20:21:48,815 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd, eba09a860a3a6c62a1b5d86497c767fa 2024-11-07T20:21:48,953 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa/.tmp/testFamily/528859907a1f4e07882a9ecc2656c1a5 2024-11-07T20:21:48,960 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa/.tmp/testFamily/528859907a1f4e07882a9ecc2656c1a5 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa/testFamily/528859907a1f4e07882a9ecc2656c1a5 2024-11-07T20:21:48,967 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa/testFamily/528859907a1f4e07882a9ecc2656c1a5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:48,968 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eba09a860a3a6c62a1b5d86497c767fa in 435ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:48,972 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/eba09a860a3a6c62a1b5d86497c767fa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:48,973 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:21:48,973 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for eba09a860a3a6c62a1b5d86497c767fa: 2024-11-07T20:21:48,973 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731010801750.eba09a860a3a6c62a1b5d86497c767fa. 2024-11-07T20:21:48,973 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e9d6a79c1b635ab184225c342afd64cd, disabling compactions & flushes 2024-11-07T20:21:48,973 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:21:48,973 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:21:48,973 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. after waiting 0 ms 2024-11-07T20:21:48,973 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:21:48,973 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e9d6a79c1b635ab184225c342afd64cd 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:48,989 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd/.tmp/testFamily/a5f4d21578504e6b8d006dcb16b6fcf9 is 61, key is row-0/testFamily:testQualifier/1731010718373/Put/seqid=0 2024-11-07T20:21:48,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741912_1088 (size=5049) 2024-11-07T20:21:49,016 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:21:49,216 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2, e9d6a79c1b635ab184225c342afd64cd 2024-11-07T20:21:49,394 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd/.tmp/testFamily/a5f4d21578504e6b8d006dcb16b6fcf9 2024-11-07T20:21:49,402 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd/.tmp/testFamily/a5f4d21578504e6b8d006dcb16b6fcf9 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd/testFamily/a5f4d21578504e6b8d006dcb16b6fcf9 2024-11-07T20:21:49,407 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd/testFamily/a5f4d21578504e6b8d006dcb16b6fcf9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:49,408 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e9d6a79c1b635ab184225c342afd64cd in 435ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:49,412 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/e9d6a79c1b635ab184225c342afd64cd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:49,413 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:21:49,413 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e9d6a79c1b635ab184225c342afd64cd: 2024-11-07T20:21:49,413 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731010715350.e9d6a79c1b635ab184225c342afd64cd. 2024-11-07T20:21:49,413 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 5dbcc8e35a32fe336d087243233e5f12, disabling compactions & flushes 2024-11-07T20:21:49,413 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:21:49,413 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:21:49,413 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. after waiting 0 ms 2024-11-07T20:21:49,413 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:21:49,413 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 5dbcc8e35a32fe336d087243233e5f12 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-07T20:21:49,416 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:49,429 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12/.tmp/info/e15475e78764498d83b2158968ad1f1c is 45, key is default/info:d/1731010711976/Put/seqid=0 2024-11-07T20:21:49,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741913_1089 (size=5037) 2024-11-07T20:21:49,616 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:49,816 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 11 regions to close 2024-11-07T20:21:49,817 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 5dbcc8e35a32fe336d087243233e5f12, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:49,834 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12/.tmp/info/e15475e78764498d83b2158968ad1f1c 2024-11-07T20:21:49,841 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12/.tmp/info/e15475e78764498d83b2158968ad1f1c as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12/info/e15475e78764498d83b2158968ad1f1c 2024-11-07T20:21:49,847 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12/info/e15475e78764498d83b2158968ad1f1c, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T20:21:49,848 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 5dbcc8e35a32fe336d087243233e5f12 in 435ms, sequenceid=6, compaction requested=false 2024-11-07T20:21:49,852 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/namespace/5dbcc8e35a32fe336d087243233e5f12/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T20:21:49,852 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:21:49,852 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 5dbcc8e35a32fe336d087243233e5f12: 2024-11-07T20:21:49,852 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731010710626.5dbcc8e35a32fe336d087243233e5f12. 2024-11-07T20:21:49,852 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 80eec78f120d1efdd45815dfc102e429, disabling compactions & flushes 2024-11-07T20:21:49,852 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:49,852 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:49,852 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. after waiting 0 ms 2024-11-07T20:21:49,852 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:49,852 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 80eec78f120d1efdd45815dfc102e429 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:49,868 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429/.tmp/testFamily/ee279c100c234b3aa661a441619a433e is 61, key is row-0/testFamily:testQualifier/1731010886716/Put/seqid=0 2024-11-07T20:21:49,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741914_1090 (size=5049) 2024-11-07T20:21:50,017 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:50,217 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, 80eec78f120d1efdd45815dfc102e429, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:50,272 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429/.tmp/testFamily/ee279c100c234b3aa661a441619a433e 2024-11-07T20:21:50,279 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429/.tmp/testFamily/ee279c100c234b3aa661a441619a433e as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429/testFamily/ee279c100c234b3aa661a441619a433e 2024-11-07T20:21:50,284 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429/testFamily/ee279c100c234b3aa661a441619a433e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:50,285 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 80eec78f120d1efdd45815dfc102e429 in 433ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:50,289 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_1_-true/80eec78f120d1efdd45815dfc102e429/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:50,289 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:50,289 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 80eec78f120d1efdd45815dfc102e429: 2024-11-07T20:21:50,289 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731010883700.80eec78f120d1efdd45815dfc102e429. 2024-11-07T20:21:50,289 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing dd525a3c56e204c238b5565ab48afa57, disabling compactions & flushes 2024-11-07T20:21:50,290 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:21:50,290 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:21:50,290 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. after waiting 0 ms 2024-11-07T20:21:50,290 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:21:50,290 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing dd525a3c56e204c238b5565ab48afa57 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:50,305 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57/.tmp/testFamily/f41c94238dfa4415a29d77b450796d08 is 61, key is row-0/testFamily:testQualifier/1731010715232/Put/seqid=0 2024-11-07T20:21:50,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741915_1091 (size=5049) 2024-11-07T20:21:50,417 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:50,618 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, dd525a3c56e204c238b5565ab48afa57, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:50,710 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57/.tmp/testFamily/f41c94238dfa4415a29d77b450796d08 2024-11-07T20:21:50,716 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57/.tmp/testFamily/f41c94238dfa4415a29d77b450796d08 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57/testFamily/f41c94238dfa4415a29d77b450796d08 2024-11-07T20:21:50,721 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57/testFamily/f41c94238dfa4415a29d77b450796d08, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:50,722 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dd525a3c56e204c238b5565ab48afa57 in 432ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:50,726 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/dd525a3c56e204c238b5565ab48afa57/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:50,726 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:21:50,726 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for dd525a3c56e204c238b5565ab48afa57: 2024-11-07T20:21:50,726 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731010712169.dd525a3c56e204c238b5565ab48afa57. 2024-11-07T20:21:50,726 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c3c1af47e03021f6356fe5780cb33e13, disabling compactions & flushes 2024-11-07T20:21:50,727 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:21:50,727 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:21:50,727 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. after waiting 0 ms 2024-11-07T20:21:50,727 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:21:50,727 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c3c1af47e03021f6356fe5780cb33e13 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:50,742 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13/.tmp/testFamily/a8dffafad9104a0d8528114830e20e27 is 61, key is row-0/testFamily:testQualifier/1731010798705/Put/seqid=0 2024-11-07T20:21:50,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741916_1092 (size=5049) 2024-11-07T20:21:50,818 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 8 regions to close 2024-11-07T20:21:50,818 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1603): Online Regions={c3c1af47e03021f6356fe5780cb33e13=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13., 2683d6dd943cbb13e85b92ced21cdb05=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05., acbc84b2cbff5d121607335cd7f60cb9=hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9., 45f0f6007cab85b2d5717f89f96bb49b=testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b., aa9aeb512dbdd6d51b6c45aacf3be68c=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., c20367372a9444f76086b80bb0cd5335=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., d1553686ddc24d1772f86227f55ddbce=testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce., e176f56c01b42e1dc5198946a56951a2=testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.} 2024-11-07T20:21:50,818 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:51,018 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, c3c1af47e03021f6356fe5780cb33e13, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:51,146 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13/.tmp/testFamily/a8dffafad9104a0d8528114830e20e27 2024-11-07T20:21:51,153 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13/.tmp/testFamily/a8dffafad9104a0d8528114830e20e27 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13/testFamily/a8dffafad9104a0d8528114830e20e27 2024-11-07T20:21:51,158 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13/testFamily/a8dffafad9104a0d8528114830e20e27, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:51,159 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c3c1af47e03021f6356fe5780cb33e13 in 432ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:51,163 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/c3c1af47e03021f6356fe5780cb33e13/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:51,163 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:21:51,163 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c3c1af47e03021f6356fe5780cb33e13: 2024-11-07T20:21:51,163 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731010795687.c3c1af47e03021f6356fe5780cb33e13. 2024-11-07T20:21:51,164 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2683d6dd943cbb13e85b92ced21cdb05, disabling compactions & flushes 2024-11-07T20:21:51,164 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:21:51,164 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:21:51,164 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. after waiting 0 ms 2024-11-07T20:21:51,164 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:21:51,164 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2683d6dd943cbb13e85b92ced21cdb05 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:51,179 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05/.tmp/testFamily/7986c36ae3a14458840c2014028d1719 is 61, key is row-0/testFamily:testQualifier/1731010807838/Put/seqid=0 2024-11-07T20:21:51,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741917_1093 (size=5049) 2024-11-07T20:21:51,218 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:51,419 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 2683d6dd943cbb13e85b92ced21cdb05, 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:51,583 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05/.tmp/testFamily/7986c36ae3a14458840c2014028d1719 2024-11-07T20:21:51,590 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05/.tmp/testFamily/7986c36ae3a14458840c2014028d1719 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05/testFamily/7986c36ae3a14458840c2014028d1719 2024-11-07T20:21:51,595 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05/testFamily/7986c36ae3a14458840c2014028d1719, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:51,596 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2683d6dd943cbb13e85b92ced21cdb05 in 432ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:51,600 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/2683d6dd943cbb13e85b92ced21cdb05/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:51,601 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:21:51,601 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2683d6dd943cbb13e85b92ced21cdb05: 2024-11-07T20:21:51,601 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731010804820.2683d6dd943cbb13e85b92ced21cdb05. 2024-11-07T20:21:51,601 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing acbc84b2cbff5d121607335cd7f60cb9, disabling compactions & flushes 2024-11-07T20:21:51,601 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:21:51,601 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:21:51,601 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. after waiting 0 ms 2024-11-07T20:21:51,601 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:21:51,601 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing acbc84b2cbff5d121607335cd7f60cb9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:51,616 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9/.tmp/testFamily/42cd1f0c697f4c409aa9610c0fdeb83b is 61, key is row-0/testFamily:testQualifier/1731010789628/Put/seqid=0 2024-11-07T20:21:51,619 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741918_1094 (size=5049) 2024-11-07T20:21:51,819 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 6 regions to close 2024-11-07T20:21:51,819 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1603): Online Regions={acbc84b2cbff5d121607335cd7f60cb9=hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9., 45f0f6007cab85b2d5717f89f96bb49b=testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b., aa9aeb512dbdd6d51b6c45aacf3be68c=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., c20367372a9444f76086b80bb0cd5335=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., d1553686ddc24d1772f86227f55ddbce=testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce., e176f56c01b42e1dc5198946a56951a2=testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.} 2024-11-07T20:21:51,819 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:52,019 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, acbc84b2cbff5d121607335cd7f60cb9, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:52,020 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9/.tmp/testFamily/42cd1f0c697f4c409aa9610c0fdeb83b 2024-11-07T20:21:52,027 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9/.tmp/testFamily/42cd1f0c697f4c409aa9610c0fdeb83b as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9/testFamily/42cd1f0c697f4c409aa9610c0fdeb83b 2024-11-07T20:21:52,032 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9/testFamily/42cd1f0c697f4c409aa9610c0fdeb83b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:52,033 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for acbc84b2cbff5d121607335cd7f60cb9 in 432ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:52,037 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/hbase/testMetaScanTimeoutOnNext_0_-false/acbc84b2cbff5d121607335cd7f60cb9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:52,038 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:21:52,038 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for acbc84b2cbff5d121607335cd7f60cb9: 2024-11-07T20:21:52,038 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731010786604.acbc84b2cbff5d121607335cd7f60cb9. 2024-11-07T20:21:52,038 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 45f0f6007cab85b2d5717f89f96bb49b, disabling compactions & flushes 2024-11-07T20:21:52,038 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:21:52,038 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:21:52,038 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. after waiting 0 ms 2024-11-07T20:21:52,038 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:21:52,038 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 45f0f6007cab85b2d5717f89f96bb49b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:52,061 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b/.tmp/testFamily/1da64a6323db4bc484adba43584575a2 is 61, key is row-0/testFamily:testQualifier/1731010760740/Put/seqid=0 2024-11-07T20:21:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741919_1095 (size=5049) 2024-11-07T20:21:52,219 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:52,420 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on 45f0f6007cab85b2d5717f89f96bb49b, aa9aeb512dbdd6d51b6c45aacf3be68c, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:52,466 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b/.tmp/testFamily/1da64a6323db4bc484adba43584575a2 2024-11-07T20:21:52,472 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b/.tmp/testFamily/1da64a6323db4bc484adba43584575a2 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b/testFamily/1da64a6323db4bc484adba43584575a2 2024-11-07T20:21:52,477 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b/testFamily/1da64a6323db4bc484adba43584575a2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:52,478 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 45f0f6007cab85b2d5717f89f96bb49b in 440ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:52,482 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnClose_0_-false/45f0f6007cab85b2d5717f89f96bb49b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:52,483 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:21:52,483 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 45f0f6007cab85b2d5717f89f96bb49b: 2024-11-07T20:21:52,483 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1731010759715.45f0f6007cab85b2d5717f89f96bb49b. 2024-11-07T20:21:52,483 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing aa9aeb512dbdd6d51b6c45aacf3be68c, disabling compactions & flushes 2024-11-07T20:21:52,483 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:21:52,483 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:21:52,483 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. after waiting 0 ms 2024-11-07T20:21:52,483 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:21:52,483 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing aa9aeb512dbdd6d51b6c45aacf3be68c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:52,498 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c/.tmp/testFamily/f3a3abc726184464b43d269390d0d7a4 is 61, key is row-0/testFamily:testQualifier/1731010744950/Put/seqid=0 2024-11-07T20:21:52,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741920_1096 (size=5049) 2024-11-07T20:21:52,620 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on aa9aeb512dbdd6d51b6c45aacf3be68c, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:52,820 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close 2024-11-07T20:21:52,820 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1603): Online Regions={aa9aeb512dbdd6d51b6c45aacf3be68c=testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c., c20367372a9444f76086b80bb0cd5335=testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335., d1553686ddc24d1772f86227f55ddbce=testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce., e176f56c01b42e1dc5198946a56951a2=testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.} 2024-11-07T20:21:52,820 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on aa9aeb512dbdd6d51b6c45aacf3be68c, c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:52,903 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c/.tmp/testFamily/f3a3abc726184464b43d269390d0d7a4 2024-11-07T20:21:52,909 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c/.tmp/testFamily/f3a3abc726184464b43d269390d0d7a4 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c/testFamily/f3a3abc726184464b43d269390d0d7a4 2024-11-07T20:21:52,914 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c/testFamily/f3a3abc726184464b43d269390d0d7a4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:52,915 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aa9aeb512dbdd6d51b6c45aacf3be68c in 432ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:52,918 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/aa9aeb512dbdd6d51b6c45aacf3be68c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:52,919 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:21:52,919 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for aa9aeb512dbdd6d51b6c45aacf3be68c: 2024-11-07T20:21:52,919 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731010741927.aa9aeb512dbdd6d51b6c45aacf3be68c. 2024-11-07T20:21:52,919 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c20367372a9444f76086b80bb0cd5335, disabling compactions & flushes 2024-11-07T20:21:52,919 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:21:52,919 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:21:52,919 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. after waiting 0 ms 2024-11-07T20:21:52,919 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:21:52,919 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c20367372a9444f76086b80bb0cd5335 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:52,934 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335/.tmp/testFamily/d56ce44fcde843afbfb53cb1bf779237 is 61, key is row-0/testFamily:testQualifier/1731010834221/Put/seqid=0 2024-11-07T20:21:52,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741921_1097 (size=5049) 2024-11-07T20:21:53,020 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:53,157 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T20:21:53,221 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on c20367372a9444f76086b80bb0cd5335, d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:53,339 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335/.tmp/testFamily/d56ce44fcde843afbfb53cb1bf779237 2024-11-07T20:21:53,345 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335/.tmp/testFamily/d56ce44fcde843afbfb53cb1bf779237 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335/testFamily/d56ce44fcde843afbfb53cb1bf779237 2024-11-07T20:21:53,350 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335/testFamily/d56ce44fcde843afbfb53cb1bf779237, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:53,351 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c20367372a9444f76086b80bb0cd5335 in 432ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:53,355 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/c20367372a9444f76086b80bb0cd5335/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:53,356 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:21:53,356 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c20367372a9444f76086b80bb0cd5335: 2024-11-07T20:21:53,356 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731010831204.c20367372a9444f76086b80bb0cd5335. 2024-11-07T20:21:53,356 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d1553686ddc24d1772f86227f55ddbce, disabling compactions & flushes 2024-11-07T20:21:53,357 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:21:53,357 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:21:53,357 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. after waiting 0 ms 2024-11-07T20:21:53,357 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:21:53,357 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d1553686ddc24d1772f86227f55ddbce 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:53,372 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce/.tmp/testFamily/621656ec523b4eb4a00cef2c85f37635 is 61, key is row-0/testFamily:testQualifier/1731010749037/Put/seqid=0 2024-11-07T20:21:53,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741922_1098 (size=5049) 2024-11-07T20:21:53,421 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:53,621 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on d1553686ddc24d1772f86227f55ddbce, e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:53,777 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce/.tmp/testFamily/621656ec523b4eb4a00cef2c85f37635 2024-11-07T20:21:53,784 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce/.tmp/testFamily/621656ec523b4eb4a00cef2c85f37635 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce/testFamily/621656ec523b4eb4a00cef2c85f37635 2024-11-07T20:21:53,790 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce/testFamily/621656ec523b4eb4a00cef2c85f37635, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:53,791 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d1553686ddc24d1772f86227f55ddbce in 434ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:53,795 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_0_-false/d1553686ddc24d1772f86227f55ddbce/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:53,796 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:21:53,796 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d1553686ddc24d1772f86227f55ddbce: 2024-11-07T20:21:53,796 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731010748010.d1553686ddc24d1772f86227f55ddbce. 2024-11-07T20:21:53,796 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e176f56c01b42e1dc5198946a56951a2, disabling compactions & flushes 2024-11-07T20:21:53,796 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:21:53,796 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:21:53,796 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. after waiting 0 ms 2024-11-07T20:21:53,796 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:21:53,796 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e176f56c01b42e1dc5198946a56951a2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-07T20:21:53,818 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2/.tmp/testFamily/6e47aa45fc7c4acc91af9b4ec21c5597 is 61, key is row-0/testFamily:testQualifier/1731010838284/Put/seqid=0 2024-11-07T20:21:53,821 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1599): Waiting on 1 regions to close 2024-11-07T20:21:53,821 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1603): Online Regions={e176f56c01b42e1dc5198946a56951a2=testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2.} 2024-11-07T20:21:53,821 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741923_1099 (size=5049) 2024-11-07T20:21:54,022 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:54,222 DEBUG [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1629): Waiting on e176f56c01b42e1dc5198946a56951a2 2024-11-07T20:21:54,223 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2/.tmp/testFamily/6e47aa45fc7c4acc91af9b4ec21c5597 2024-11-07T20:21:54,229 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2/.tmp/testFamily/6e47aa45fc7c4acc91af9b4ec21c5597 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2/testFamily/6e47aa45fc7c4acc91af9b4ec21c5597 2024-11-07T20:21:54,235 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2/testFamily/6e47aa45fc7c4acc91af9b4ec21c5597, entries=4, sequenceid=8, filesize=4.9 K 2024-11-07T20:21:54,235 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e176f56c01b42e1dc5198946a56951a2 in 439ms, sequenceid=8, compaction requested=false 2024-11-07T20:21:54,239 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/data/default/testNormalScanTimeoutOnNextAsync_1_-true/e176f56c01b42e1dc5198946a56951a2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-07T20:21:54,240 INFO [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:21:54,240 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e176f56c01b42e1dc5198946a56951a2: 2024-11-07T20:21:54,240 DEBUG [RS_CLOSE_REGION-regionserver/15c0e1002410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731010837270.e176f56c01b42e1dc5198946a56951a2. 2024-11-07T20:21:54,422 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1250): stopping server 15c0e1002410,32779,1731010677532; all regions closed. 2024-11-07T20:21:54,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741834_1010 (size=56731) 2024-11-07T20:21:54,432 DEBUG [RS:0;15c0e1002410:32779 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/oldWALs 2024-11-07T20:21:54,432 INFO [RS:0;15c0e1002410:32779 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 15c0e1002410%2C32779%2C1731010677532.meta:.meta(num 1731010710386) 2024-11-07T20:21:54,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741833_1009 (size=60562) 2024-11-07T20:21:54,437 DEBUG [RS:0;15c0e1002410:32779 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/oldWALs 2024-11-07T20:21:54,437 INFO [RS:0;15c0e1002410:32779 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 15c0e1002410%2C32779%2C1731010677532:(num 1731010709932) 2024-11-07T20:21:54,437 DEBUG [RS:0;15c0e1002410:32779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:54,437 INFO [RS:0;15c0e1002410:32779 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T20:21:54,437 INFO [RS:0;15c0e1002410:32779 {}] hbase.ChoreService(370): Chore service for: regionserver/15c0e1002410:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T20:21:54,438 INFO [regionserver/15c0e1002410:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T20:21:54,439 INFO [RS:0;15c0e1002410:32779 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:32779 2024-11-07T20:21:54,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/15c0e1002410,32779,1731010677532 2024-11-07T20:21:54,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T20:21:54,444 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [15c0e1002410,32779,1731010677532] 2024-11-07T20:21:54,444 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 15c0e1002410,32779,1731010677532; numProcessing=1 2024-11-07T20:21:54,445 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/15c0e1002410,32779,1731010677532 already deleted, retry=false 2024-11-07T20:21:54,445 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 15c0e1002410,32779,1731010677532 expired; onlineServers=0 2024-11-07T20:21:54,445 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '15c0e1002410,46471,1731010676369' ***** 2024-11-07T20:21:54,445 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T20:21:54,445 DEBUG [M:0;15c0e1002410:46471 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a9233b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=15c0e1002410/172.17.0.2:0 2024-11-07T20:21:54,446 INFO [M:0;15c0e1002410:46471 {}] regionserver.HRegionServer(1224): stopping server 15c0e1002410,46471,1731010676369 2024-11-07T20:21:54,446 INFO [M:0;15c0e1002410:46471 {}] regionserver.HRegionServer(1250): stopping server 15c0e1002410,46471,1731010676369; all regions closed. 2024-11-07T20:21:54,446 DEBUG [M:0;15c0e1002410:46471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T20:21:54,446 DEBUG [M:0;15c0e1002410:46471 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T20:21:54,446 DEBUG [M:0;15c0e1002410:46471 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T20:21:54,446 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T20:21:54,446 DEBUG [master/15c0e1002410:0:becomeActiveMaster-HFileCleaner.large.0-1731010679734 {}] cleaner.HFileCleaner(306): Exit Thread[master/15c0e1002410:0:becomeActiveMaster-HFileCleaner.large.0-1731010679734,5,FailOnTimeoutGroup] 2024-11-07T20:21:54,446 DEBUG [master/15c0e1002410:0:becomeActiveMaster-HFileCleaner.small.0-1731010679737 {}] cleaner.HFileCleaner(306): Exit Thread[master/15c0e1002410:0:becomeActiveMaster-HFileCleaner.small.0-1731010679737,5,FailOnTimeoutGroup] 2024-11-07T20:21:54,446 INFO [M:0;15c0e1002410:46471 {}] hbase.ChoreService(370): Chore service for: master/15c0e1002410:0 had [] on shutdown 2024-11-07T20:21:54,446 DEBUG [M:0;15c0e1002410:46471 {}] master.HMaster(1733): Stopping service threads 2024-11-07T20:21:54,446 INFO [M:0;15c0e1002410:46471 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T20:21:54,447 INFO [M:0;15c0e1002410:46471 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T20:21:54,447 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T20:21:54,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T20:21:54,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T20:21:54,447 DEBUG [M:0;15c0e1002410:46471 {}] zookeeper.ZKUtil(347): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T20:21:54,447 WARN [M:0;15c0e1002410:46471 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T20:21:54,447 INFO [M:0;15c0e1002410:46471 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-07T20:21:54,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T20:21:54,448 INFO [M:0;15c0e1002410:46471 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T20:21:54,448 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T20:21:54,448 INFO [M:0;15c0e1002410:46471 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:21:54,448 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:21:54,448 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T20:21:54,448 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:21:54,448 INFO [M:0;15c0e1002410:46471 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.20 KB heapSize=519.63 KB 2024-11-07T20:21:54,466 DEBUG [M:0;15c0e1002410:46471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10a49d785c0d4a02ba4800e4bd5666c8 is 82, key is hbase:meta,,1/info:regioninfo/1731010710522/Put/seqid=0 2024-11-07T20:21:54,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741924_1100 (size=5672) 2024-11-07T20:21:54,492 INFO [regionserver/15c0e1002410:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T20:21:54,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T20:21:54,544 INFO [RS:0;15c0e1002410:32779 {}] regionserver.HRegionServer(1307): Exiting; stopping=15c0e1002410,32779,1731010677532; zookeeper connection closed. 2024-11-07T20:21:54,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32779-0x101846238170001, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T20:21:54,544 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a74b3e2 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a74b3e2 2024-11-07T20:21:54,545 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T20:21:54,871 INFO [M:0;15c0e1002410:46471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10a49d785c0d4a02ba4800e4bd5666c8 2024-11-07T20:21:54,894 DEBUG [M:0;15c0e1002410:46471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a0c7fc5c97884d58a6eb33ec397e5ad8 is 830, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1731010713397/Put/seqid=0 2024-11-07T20:21:54,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741925_1101 (size=30262) 2024-11-07T20:21:55,282 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T20:21:55,300 INFO [M:0;15c0e1002410:46471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=447.64 KB at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a0c7fc5c97884d58a6eb33ec397e5ad8 2024-11-07T20:21:55,305 INFO [M:0;15c0e1002410:46471 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a0c7fc5c97884d58a6eb33ec397e5ad8 2024-11-07T20:21:55,320 DEBUG [M:0;15c0e1002410:46471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c79c9ec05cf4bfc9910ff21287e9d81 is 69, key is 15c0e1002410,32779,1731010677532/rs:state/1731010679768/Put/seqid=0 2024-11-07T20:21:55,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741926_1102 (size=5156) 2024-11-07T20:21:55,725 INFO [M:0;15c0e1002410:46471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c79c9ec05cf4bfc9910ff21287e9d81 2024-11-07T20:21:55,731 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10a49d785c0d4a02ba4800e4bd5666c8 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10a49d785c0d4a02ba4800e4bd5666c8 2024-11-07T20:21:55,736 INFO [M:0;15c0e1002410:46471 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10a49d785c0d4a02ba4800e4bd5666c8, entries=8, sequenceid=830, filesize=5.5 K 2024-11-07T20:21:55,737 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a0c7fc5c97884d58a6eb33ec397e5ad8 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a0c7fc5c97884d58a6eb33ec397e5ad8 2024-11-07T20:21:55,742 INFO [M:0;15c0e1002410:46471 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a0c7fc5c97884d58a6eb33ec397e5ad8 2024-11-07T20:21:55,742 INFO [M:0;15c0e1002410:46471 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a0c7fc5c97884d58a6eb33ec397e5ad8, entries=92, sequenceid=830, filesize=29.6 K 2024-11-07T20:21:55,743 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0c79c9ec05cf4bfc9910ff21287e9d81 as hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0c79c9ec05cf4bfc9910ff21287e9d81 2024-11-07T20:21:55,748 INFO [M:0;15c0e1002410:46471 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:39675/user/jenkins/test-data/f3436b9f-3c4a-270b-4f75-dd7d6c9e1de7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0c79c9ec05cf4bfc9910ff21287e9d81, entries=1, sequenceid=830, filesize=5.0 K 2024-11-07T20:21:55,749 INFO [M:0;15c0e1002410:46471 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.20 KB/458952, heapSize ~519.34 KB/531800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1301ms, sequenceid=830, compaction requested=false 2024-11-07T20:21:55,750 INFO [M:0;15c0e1002410:46471 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T20:21:55,750 DEBUG [M:0;15c0e1002410:46471 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-07T20:21:55,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33209 is added to blk_1073741830_1006 (size=513821) 2024-11-07T20:21:55,753 INFO [M:0;15c0e1002410:46471 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-07T20:21:55,753 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-07T20:21:55,753 INFO [M:0;15c0e1002410:46471 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:46471 2024-11-07T20:21:55,755 DEBUG [M:0;15c0e1002410:46471 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/15c0e1002410,46471,1731010676369 already deleted, retry=false 2024-11-07T20:21:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T20:21:55,856 INFO [M:0;15c0e1002410:46471 {}] regionserver.HRegionServer(1307): Exiting; stopping=15c0e1002410,46471,1731010676369; zookeeper connection closed. 2024-11-07T20:21:55,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46471-0x101846238170000, quorum=127.0.0.1:59122, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T20:21:55,861 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1988537a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T20:21:55,864 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f9e40a8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T20:21:55,864 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T20:21:55,864 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@750f55{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T20:21:55,864 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cf34ac6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/hadoop.log.dir/,STOPPED} 2024-11-07T20:21:55,867 WARN [BP-181548129-172.17.0.2-1731010672065 heartbeating to localhost/127.0.0.1:39675 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T20:21:55,867 WARN [BP-181548129-172.17.0.2-1731010672065 heartbeating to localhost/127.0.0.1:39675 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-181548129-172.17.0.2-1731010672065 (Datanode Uuid 4751e2ac-fcba-46f5-8dda-83b57dedad2a) service to localhost/127.0.0.1:39675 2024-11-07T20:21:55,868 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83/dfs/data/data1/current/BP-181548129-172.17.0.2-1731010672065 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T20:21:55,869 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/cluster_46b1d0b4-96d7-815d-6260-ff41de028e83/dfs/data/data2/current/BP-181548129-172.17.0.2-1731010672065 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T20:21:55,869 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T20:21:55,869 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T20:21:55,869 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T20:21:55,875 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4f422bdf{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T20:21:55,876 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c457aa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T20:21:55,876 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T20:21:55,876 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4803ff98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T20:21:55,876 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1857c056{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/35a8a5c6-00a6-353c-770a-5e2e56adf305/hadoop.log.dir/,STOPPED} 2024-11-07T20:21:55,886 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-07T20:21:55,919 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down