2024-11-21 17:41:34,049 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-21 17:41:34,067 main DEBUG Took 0.016501 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 17:41:34,067 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 17:41:34,068 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 17:41:34,069 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 17:41:34,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,077 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 17:41:34,089 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,092 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,092 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,094 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,095 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,096 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,097 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,098 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,099 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,100 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,100 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 17:41:34,101 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,101 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 17:41:34,103 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 17:41:34,104 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 17:41:34,106 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 17:41:34,106 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 17:41:34,107 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 17:41:34,108 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 17:41:34,116 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 17:41:34,118 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 17:41:34,120 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 17:41:34,120 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 17:41:34,121 main DEBUG createAppenders(={Console}) 2024-11-21 17:41:34,122 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-21 17:41:34,122 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-21 17:41:34,122 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-21 17:41:34,123 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 17:41:34,123 main DEBUG OutputStream closed 2024-11-21 17:41:34,123 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 17:41:34,124 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 17:41:34,124 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-21 17:41:34,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 17:41:34,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 17:41:34,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 17:41:34,200 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 17:41:34,200 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 17:41:34,201 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 17:41:34,201 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 17:41:34,202 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 17:41:34,202 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 17:41:34,202 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 17:41:34,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 17:41:34,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 17:41:34,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 17:41:34,204 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 17:41:34,204 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 17:41:34,204 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 17:41:34,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 17:41:34,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 17:41:34,208 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 17:41:34,208 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-21 17:41:34,208 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 17:41:34,209 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-21T17:41:34,221 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-21 17:41:34,224 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 17:41:34,224 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T17:41:34,462 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9 2024-11-21T17:41:34,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T17:41:34,495 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127, deleteOnExit=true 2024-11-21T17:41:34,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T17:41:34,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/test.cache.data in system properties and HBase conf 2024-11-21T17:41:34,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T17:41:34,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/hadoop.log.dir in system properties and HBase conf 2024-11-21T17:41:34,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T17:41:34,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T17:41:34,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T17:41:34,582 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T17:41:34,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T17:41:34,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T17:41:34,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T17:41:34,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T17:41:34,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T17:41:34,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T17:41:34,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T17:41:34,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T17:41:34,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T17:41:34,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T17:41:34,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/nfs.dump.dir in system properties and HBase conf 2024-11-21T17:41:34,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/java.io.tmpdir in system properties and HBase conf 2024-11-21T17:41:34,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T17:41:34,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T17:41:34,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T17:41:35,397 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T17:41:35,464 INFO [Time-limited test {}] log.Log(170): Logging initialized @2055ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T17:41:35,528 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T17:41:35,585 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T17:41:35,606 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T17:41:35,606 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T17:41:35,608 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T17:41:35,621 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T17:41:35,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/hadoop.log.dir/,AVAILABLE} 2024-11-21T17:41:35,625 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T17:41:35,804 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/java.io.tmpdir/jetty-localhost-44723-hadoop-hdfs-3_4_1-tests_jar-_-any-12878499688996866421/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T17:41:35,809 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:44723} 2024-11-21T17:41:35,809 INFO [Time-limited test {}] server.Server(415): Started @2402ms 2024-11-21T17:41:36,133 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T17:41:36,142 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T17:41:36,143 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T17:41:36,143 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T17:41:36,144 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T17:41:36,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/hadoop.log.dir/,AVAILABLE} 2024-11-21T17:41:36,145 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T17:41:36,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/java.io.tmpdir/jetty-localhost-45739-hadoop-hdfs-3_4_1-tests_jar-_-any-7681062664855226932/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T17:41:36,244 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:45739} 2024-11-21T17:41:36,245 INFO [Time-limited test {}] server.Server(415): Started @2837ms 2024-11-21T17:41:36,294 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T17:41:36,726 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127/data/data1/current/BP-330246287-172.17.0.2-1732210895193/current, will proceed with Du for space computation calculation, 2024-11-21T17:41:36,726 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127/data/data2/current/BP-330246287-172.17.0.2-1732210895193/current, will proceed with Du for space computation calculation, 2024-11-21T17:41:36,767 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T17:41:36,816 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5db7244844c542b with lease ID 0x7b428f91a8ea7f52: Processing first storage report for DS-c25b372d-f8ee-43cf-9943-4132ec7b817b from datanode DatanodeRegistration(127.0.0.1:33819, datanodeUuid=4a17df3d-ecb2-49d6-bbf2-3c24e42ae4a8, infoPort=35731, infoSecurePort=0, ipcPort=33927, storageInfo=lv=-57;cid=testClusterID;nsid=1561730071;c=1732210895193) 2024-11-21T17:41:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5db7244844c542b with lease ID 0x7b428f91a8ea7f52: from storage DS-c25b372d-f8ee-43cf-9943-4132ec7b817b node DatanodeRegistration(127.0.0.1:33819, datanodeUuid=4a17df3d-ecb2-49d6-bbf2-3c24e42ae4a8, infoPort=35731, infoSecurePort=0, ipcPort=33927, storageInfo=lv=-57;cid=testClusterID;nsid=1561730071;c=1732210895193), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T17:41:36,817 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5db7244844c542b with lease ID 0x7b428f91a8ea7f52: Processing first storage report for DS-5f72e8fa-f59c-4444-9358-1e796febf7d0 from datanode DatanodeRegistration(127.0.0.1:33819, datanodeUuid=4a17df3d-ecb2-49d6-bbf2-3c24e42ae4a8, infoPort=35731, infoSecurePort=0, ipcPort=33927, storageInfo=lv=-57;cid=testClusterID;nsid=1561730071;c=1732210895193) 2024-11-21T17:41:36,818 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5db7244844c542b with lease ID 0x7b428f91a8ea7f52: from storage DS-5f72e8fa-f59c-4444-9358-1e796febf7d0 node DatanodeRegistration(127.0.0.1:33819, datanodeUuid=4a17df3d-ecb2-49d6-bbf2-3c24e42ae4a8, infoPort=35731, infoSecurePort=0, ipcPort=33927, storageInfo=lv=-57;cid=testClusterID;nsid=1561730071;c=1732210895193), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T17:41:36,852 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9 2024-11-21T17:41:36,919 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127/zookeeper_0, clientPort=56661, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T17:41:36,929 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56661 2024-11-21T17:41:36,939 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T17:41:36,941 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T17:41:37,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741825_1001 (size=7) 2024-11-21T17:41:37,561 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 with version=8 2024-11-21T17:41:37,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/hbase-staging 2024-11-21T17:41:37,641 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T17:41:37,836 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f3865e8353b4:0 server-side Connection retries=9 2024-11-21T17:41:37,844 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T17:41:37,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T17:41:37,849 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T17:41:37,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T17:41:37,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T17:41:37,984 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T17:41:38,038 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T17:41:38,047 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T17:41:38,050 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T17:41:38,073 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 132596 (auto-detected) 2024-11-21T17:41:38,074 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T17:41:38,090 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46457 2024-11-21T17:41:38,109 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46457 connecting to ZooKeeper ensemble=127.0.0.1:56661 2024-11-21T17:41:38,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464570x0, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T17:41:38,137 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46457-0x10063600c730000 connected 2024-11-21T17:41:38,164 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T17:41:38,166 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T17:41:38,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T17:41:38,179 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808, hbase.cluster.distributed=false 2024-11-21T17:41:38,203 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T17:41:38,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46457 2024-11-21T17:41:38,209 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46457 2024-11-21T17:41:38,209 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46457 2024-11-21T17:41:38,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46457 2024-11-21T17:41:38,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46457 2024-11-21T17:41:38,309 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f3865e8353b4:0 server-side Connection retries=9 2024-11-21T17:41:38,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T17:41:38,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T17:41:38,311 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T17:41:38,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T17:41:38,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T17:41:38,314 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T17:41:38,317 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T17:41:38,318 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46273 2024-11-21T17:41:38,320 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46273 connecting to ZooKeeper ensemble=127.0.0.1:56661 2024-11-21T17:41:38,321 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T17:41:38,324 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T17:41:38,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462730x0, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T17:41:38,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:462730x0, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T17:41:38,330 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46273-0x10063600c730001 connected 2024-11-21T17:41:38,333 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T17:41:38,342 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T17:41:38,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T17:41:38,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T17:41:38,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46273 2024-11-21T17:41:38,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46273 2024-11-21T17:41:38,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46273 2024-11-21T17:41:38,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46273 2024-11-21T17:41:38,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46273 2024-11-21T17:41:38,366 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f3865e8353b4:46457 2024-11-21T17:41:38,368 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f3865e8353b4,46457,1732210897685 2024-11-21T17:41:38,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T17:41:38,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T17:41:38,376 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f3865e8353b4,46457,1732210897685 2024-11-21T17:41:38,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T17:41:38,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:38,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:38,395 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T17:41:38,396 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f3865e8353b4,46457,1732210897685 from backup master directory 2024-11-21T17:41:38,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f3865e8353b4,46457,1732210897685 2024-11-21T17:41:38,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T17:41:38,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T17:41:38,399 WARN [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T17:41:38,400 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f3865e8353b4,46457,1732210897685 2024-11-21T17:41:38,402 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T17:41:38,403 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T17:41:38,461 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/hbase.id] with ID: a3794870-87a3-42b3-a4aa-de607dd0c74f 2024-11-21T17:41:38,462 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/.tmp/hbase.id 2024-11-21T17:41:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741826_1002 (size=42) 2024-11-21T17:41:38,879 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/.tmp/hbase.id]:[hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/hbase.id] 2024-11-21T17:41:38,926 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T17:41:38,931 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T17:41:38,948 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-21T17:41:38,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:38,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:38,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741827_1003 (size=196) 2024-11-21T17:41:39,385 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:41:39,387 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T17:41:39,399 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T17:41:39,403 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T17:41:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741828_1004 (size=1189) 2024-11-21T17:41:39,855 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store 2024-11-21T17:41:39,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741829_1005 (size=34) 2024-11-21T17:41:40,282 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T17:41:40,285 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:41:40,287 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T17:41:40,287 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:41:40,287 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:41:40,289 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T17:41:40,289 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:41:40,289 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:41:40,291 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732210900286Disabling compacts and flushes for region at 1732210900286Disabling writes for close at 1732210900289 (+3 ms)Writing region close event to WAL at 1732210900289Closed at 1732210900289 2024-11-21T17:41:40,294 WARN [master/f3865e8353b4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/.initializing 2024-11-21T17:41:40,294 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/WALs/f3865e8353b4,46457,1732210897685 2024-11-21T17:41:40,301 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T17:41:40,315 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f3865e8353b4%2C46457%2C1732210897685, suffix=, logDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/WALs/f3865e8353b4,46457,1732210897685, archiveDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/oldWALs, maxLogs=10 2024-11-21T17:41:40,336 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/WALs/f3865e8353b4,46457,1732210897685/f3865e8353b4%2C46457%2C1732210897685.1732210900320, exclude list is [], retry=0 2024-11-21T17:41:40,354 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33819,DS-c25b372d-f8ee-43cf-9943-4132ec7b817b,DISK] 2024-11-21T17:41:40,358 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T17:41:40,395 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/WALs/f3865e8353b4,46457,1732210897685/f3865e8353b4%2C46457%2C1732210897685.1732210900320 2024-11-21T17:41:40,396 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731)] 2024-11-21T17:41:40,396 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:41:40,397 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:41:40,400 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,400 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T17:41:40,458 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:40,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:41:40,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,465 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T17:41:40,465 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:40,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:41:40,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T17:41:40,469 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:40,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:41:40,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,473 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T17:41:40,473 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:40,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:41:40,474 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,477 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,479 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,483 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,484 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,487 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T17:41:40,491 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T17:41:40,495 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:41:40,497 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74728745, jitterRate=0.11354507505893707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T17:41:40,504 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732210900411Initializing all the Stores at 1732210900413 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732210900414 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210900415 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210900415Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210900415Cleaning up temporary data from old regions at 1732210900484 (+69 ms)Region opened successfully at 1732210900504 (+20 ms) 2024-11-21T17:41:40,505 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T17:41:40,534 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2fed89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f3865e8353b4/172.17.0.2:0 2024-11-21T17:41:40,561 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T17:41:40,571 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T17:41:40,572 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T17:41:40,574 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T17:41:40,576 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T17:41:40,580 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T17:41:40,580 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T17:41:40,604 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T17:41:40,612 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T17:41:40,613 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T17:41:40,616 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T17:41:40,618 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T17:41:40,619 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T17:41:40,621 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T17:41:40,624 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T17:41:40,625 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T17:41:40,627 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T17:41:40,628 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T17:41:40,643 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T17:41:40,644 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T17:41:40,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T17:41:40,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T17:41:40,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:40,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:40,650 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f3865e8353b4,46457,1732210897685, sessionid=0x10063600c730000, setting cluster-up flag (Was=false) 2024-11-21T17:41:40,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:40,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:40,664 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T17:41:40,665 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f3865e8353b4,46457,1732210897685 2024-11-21T17:41:40,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:40,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:41:40,674 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T17:41:40,675 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f3865e8353b4,46457,1732210897685 2024-11-21T17:41:40,681 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T17:41:40,747 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T17:41:40,755 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T17:41:40,758 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(746): ClusterId : a3794870-87a3-42b3-a4aa-de607dd0c74f 2024-11-21T17:41:40,760 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T17:41:40,762 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T17:41:40,765 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T17:41:40,765 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T17:41:40,768 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T17:41:40,768 DEBUG [RS:0;f3865e8353b4:46273 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361f498d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f3865e8353b4/172.17.0.2:0 2024-11-21T17:41:40,768 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f3865e8353b4,46457,1732210897685 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T17:41:40,776 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f3865e8353b4:0, corePoolSize=5, maxPoolSize=5 2024-11-21T17:41:40,776 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f3865e8353b4:0, corePoolSize=5, maxPoolSize=5 2024-11-21T17:41:40,776 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f3865e8353b4:0, corePoolSize=5, maxPoolSize=5 2024-11-21T17:41:40,776 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f3865e8353b4:0, corePoolSize=5, maxPoolSize=5 2024-11-21T17:41:40,777 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f3865e8353b4:0, corePoolSize=10, maxPoolSize=10 2024-11-21T17:41:40,777 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,777 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f3865e8353b4:0, corePoolSize=2, maxPoolSize=2 2024-11-21T17:41:40,777 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,778 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732210930778 2024-11-21T17:41:40,780 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T17:41:40,781 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T17:41:40,782 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T17:41:40,782 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T17:41:40,784 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T17:41:40,785 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f3865e8353b4:46273 2024-11-21T17:41:40,785 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T17:41:40,785 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T17:41:40,785 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T17:41:40,786 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,788 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:40,789 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T17:41:40,789 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T17:41:40,789 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T17:41:40,789 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T17:41:40,790 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T17:41:40,791 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T17:41:40,792 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T17:41:40,792 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(2659): reportForDuty to master=f3865e8353b4,46457,1732210897685 with port=46273, startcode=1732210898278 2024-11-21T17:41:40,794 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T17:41:40,795 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T17:41:40,800 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f3865e8353b4:0:becomeActiveMaster-HFileCleaner.large.0-1732210900796,5,FailOnTimeoutGroup] 2024-11-21T17:41:40,801 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f3865e8353b4:0:becomeActiveMaster-HFileCleaner.small.0-1732210900800,5,FailOnTimeoutGroup] 2024-11-21T17:41:40,802 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,802 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T17:41:40,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741831_1007 (size=1321) 2024-11-21T17:41:40,803 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,804 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,805 DEBUG [RS:0;f3865e8353b4:46273 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T17:41:40,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41561, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T17:41:40,847 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46457 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f3865e8353b4,46273,1732210898278 2024-11-21T17:41:40,849 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46457 {}] master.ServerManager(517): Registering regionserver=f3865e8353b4,46273,1732210898278 2024-11-21T17:41:40,863 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:41:40,863 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37327 2024-11-21T17:41:40,863 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T17:41:40,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T17:41:40,868 DEBUG [RS:0;f3865e8353b4:46273 {}] zookeeper.ZKUtil(111): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f3865e8353b4,46273,1732210898278 2024-11-21T17:41:40,868 WARN [RS:0;f3865e8353b4:46273 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T17:41:40,868 INFO [RS:0;f3865e8353b4:46273 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T17:41:40,868 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/WALs/f3865e8353b4,46273,1732210898278 2024-11-21T17:41:40,871 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f3865e8353b4,46273,1732210898278] 2024-11-21T17:41:40,899 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T17:41:40,921 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T17:41:40,926 INFO [RS:0;f3865e8353b4:46273 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T17:41:40,927 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,928 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T17:41:40,934 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T17:41:40,935 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,935 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,935 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,935 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f3865e8353b4:0, corePoolSize=2, maxPoolSize=2 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,936 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,937 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f3865e8353b4:0, corePoolSize=1, maxPoolSize=1 2024-11-21T17:41:40,937 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f3865e8353b4:0, corePoolSize=3, maxPoolSize=3 2024-11-21T17:41:40,937 DEBUG [RS:0;f3865e8353b4:46273 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f3865e8353b4:0, corePoolSize=3, maxPoolSize=3 2024-11-21T17:41:40,938 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,938 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,938 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,938 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,938 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,938 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46273,1732210898278-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T17:41:40,954 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T17:41:40,956 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46273,1732210898278-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,956 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,956 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.Replication(171): f3865e8353b4,46273,1732210898278 started 2024-11-21T17:41:40,977 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T17:41:40,978 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1482): Serving as f3865e8353b4,46273,1732210898278, RpcServer on f3865e8353b4/172.17.0.2:46273, sessionid=0x10063600c730001 2024-11-21T17:41:40,979 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T17:41:40,979 DEBUG [RS:0;f3865e8353b4:46273 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f3865e8353b4,46273,1732210898278 2024-11-21T17:41:40,979 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f3865e8353b4,46273,1732210898278' 2024-11-21T17:41:40,979 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T17:41:40,980 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T17:41:40,981 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T17:41:40,981 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T17:41:40,981 DEBUG [RS:0;f3865e8353b4:46273 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f3865e8353b4,46273,1732210898278 2024-11-21T17:41:40,981 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f3865e8353b4,46273,1732210898278' 2024-11-21T17:41:40,981 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T17:41:40,982 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T17:41:40,982 DEBUG [RS:0;f3865e8353b4:46273 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T17:41:40,983 INFO [RS:0;f3865e8353b4:46273 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T17:41:40,983 INFO [RS:0;f3865e8353b4:46273 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T17:41:41,210 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T17:41:41,211 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:41:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741832_1008 (size=32) 2024-11-21T17:41:41,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:41:41,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T17:41:41,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T17:41:41,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:41,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:41:41,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T17:41:41,235 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T17:41:41,235 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:41,236 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:41:41,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T17:41:41,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T17:41:41,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:41,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:41:41,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T17:41:41,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T17:41:41,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:41:41,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:41:41,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T17:41:41,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740 2024-11-21T17:41:41,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740 2024-11-21T17:41:41,250 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T17:41:41,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T17:41:41,252 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T17:41:41,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T17:41:41,258 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:41:41,259 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63236488, jitterRate=-0.05770289897918701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T17:41:41,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732210901224Initializing all the Stores at 1732210901227 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732210901227Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732210901227Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210901227Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732210901227Cleaning up temporary data from old regions at 1732210901251 (+24 ms)Region opened successfully at 1732210901262 (+11 ms) 2024-11-21T17:41:41,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T17:41:41,262 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T17:41:41,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T17:41:41,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T17:41:41,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T17:41:41,264 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T17:41:41,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732210901262Disabling compacts and flushes for region at 1732210901262Disabling writes for close at 1732210901262Writing region close event to WAL at 1732210901263 (+1 ms)Closed at 1732210901264 (+1 ms) 2024-11-21T17:41:41,267 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T17:41:41,267 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T17:41:41,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T17:41:41,280 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T17:41:41,282 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T17:41:41,434 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:42,435 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:43,436 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:44,438 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:45,439 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:46,440 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:47,441 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:48,036 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T17:41:48,038 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T17:41:48,443 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:49,444 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:50,445 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:51,446 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:52,447 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:53,448 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:54,449 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:55,450 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:56,452 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:57,453 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:58,454 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:41:59,455 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:00,456 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:01,457 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:02,458 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:03,460 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:04,461 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:05,462 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:06,463 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:06,848 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T17:42:07,464 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:08,465 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:09,466 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:10,467 WARN [f3865e8353b4:46457 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T17:42:10,993 INFO [RS:0;f3865e8353b4:46273 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T17:42:10,998 INFO [RS:0;f3865e8353b4:46273 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f3865e8353b4%2C46273%2C1732210898278, suffix=, logDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/WALs/f3865e8353b4,46273,1732210898278, archiveDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/oldWALs, maxLogs=32 2024-11-21T17:42:11,018 DEBUG [RS:0;f3865e8353b4:46273 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/WALs/f3865e8353b4,46273,1732210898278/f3865e8353b4%2C46273%2C1732210898278.1732210931002, exclude list is [], retry=0 2024-11-21T17:42:11,022 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33819,DS-c25b372d-f8ee-43cf-9943-4132ec7b817b,DISK] 2024-11-21T17:42:11,025 INFO [RS:0;f3865e8353b4:46273 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/WALs/f3865e8353b4,46273,1732210898278/f3865e8353b4%2C46273%2C1732210898278.1732210931002 2024-11-21T17:42:11,025 DEBUG [RS:0;f3865e8353b4:46273 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731)] 2024-11-21T17:42:11,225 DEBUG [f3865e8353b4:46457 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T17:42:11,237 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:11,244 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f3865e8353b4,46273,1732210898278, state=OPENING 2024-11-21T17:42:11,248 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T17:42:11,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:42:11,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:42:11,250 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T17:42:11,250 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T17:42:11,252 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T17:42:11,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:11,432 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T17:42:11,435 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43457, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T17:42:11,447 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T17:42:11,447 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T17:42:11,448 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T17:42:11,452 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f3865e8353b4%2C46273%2C1732210898278.meta, suffix=.meta, logDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/WALs/f3865e8353b4,46273,1732210898278, archiveDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/oldWALs, maxLogs=32 2024-11-21T17:42:11,467 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/WALs/f3865e8353b4,46273,1732210898278/f3865e8353b4%2C46273%2C1732210898278.meta.1732210931454.meta, exclude list is [], retry=0 2024-11-21T17:42:11,470 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33819,DS-c25b372d-f8ee-43cf-9943-4132ec7b817b,DISK] 2024-11-21T17:42:11,473 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/WALs/f3865e8353b4,46273,1732210898278/f3865e8353b4%2C46273%2C1732210898278.meta.1732210931454.meta 2024-11-21T17:42:11,474 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35731:35731)] 2024-11-21T17:42:11,474 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:11,476 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T17:42:11,478 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T17:42:11,482 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T17:42:11,486 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T17:42:11,487 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:11,487 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T17:42:11,487 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T17:42:11,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T17:42:11,492 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T17:42:11,492 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:11,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:42:11,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T17:42:11,495 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T17:42:11,495 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:11,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:42:11,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T17:42:11,498 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T17:42:11,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:11,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:42:11,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T17:42:11,500 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T17:42:11,500 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:11,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T17:42:11,501 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T17:42:11,503 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740 2024-11-21T17:42:11,506 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740 2024-11-21T17:42:11,509 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T17:42:11,509 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T17:42:11,510 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T17:42:11,513 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T17:42:11,515 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72244418, jitterRate=0.0765257179737091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T17:42:11,515 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T17:42:11,516 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732210931488Writing region info on filesystem at 1732210931488Initializing all the Stores at 1732210931490 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732210931490Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732210931490Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210931490Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732210931490Cleaning up temporary data from old regions at 1732210931509 (+19 ms)Running coprocessor post-open hooks at 1732210931515 (+6 ms)Region opened successfully at 1732210931516 (+1 ms) 2024-11-21T17:42:11,523 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732210931425 2024-11-21T17:42:11,534 DEBUG [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T17:42:11,534 INFO [RS_OPEN_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T17:42:11,536 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:11,539 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f3865e8353b4,46273,1732210898278, state=OPEN 2024-11-21T17:42:11,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T17:42:11,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T17:42:11,542 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T17:42:11,542 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T17:42:11,542 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:11,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T17:42:11,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f3865e8353b4,46273,1732210898278 in 289 msec 2024-11-21T17:42:11,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T17:42:11,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.2780 sec 2024-11-21T17:42:11,557 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T17:42:11,557 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T17:42:11,573 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T17:42:11,574 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f3865e8353b4,46273,1732210898278, seqNum=-1] 2024-11-21T17:42:11,592 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T17:42:11,595 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53217, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T17:42:11,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9040 sec 2024-11-21T17:42:11,615 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732210931614, completionTime=-1 2024-11-21T17:42:11,617 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T17:42:11,617 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T17:42:11,641 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-21T17:42:11,641 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732210991641 2024-11-21T17:42:11,641 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732211051641 2024-11-21T17:42:11,642 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-21T17:42:11,643 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46457,1732210897685-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T17:42:11,644 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46457,1732210897685-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T17:42:11,644 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46457,1732210897685-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T17:42:11,645 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f3865e8353b4:46457, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T17:42:11,646 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T17:42:11,646 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T17:42:11,652 DEBUG [master/f3865e8353b4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T17:42:11,670 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.270sec 2024-11-21T17:42:11,671 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T17:42:11,672 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T17:42:11,673 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T17:42:11,674 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T17:42:11,674 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T17:42:11,674 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46457,1732210897685-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T17:42:11,675 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46457,1732210897685-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T17:42:11,683 DEBUG [master/f3865e8353b4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T17:42:11,684 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T17:42:11,684 INFO [master/f3865e8353b4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f3865e8353b4,46457,1732210897685-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T17:42:11,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4225c540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T17:42:11,733 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T17:42:11,733 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T17:42:11,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f3865e8353b4,46457,-1 for getting cluster id 2024-11-21T17:42:11,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T17:42:11,747 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3794870-87a3-42b3-a4aa-de607dd0c74f' 2024-11-21T17:42:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T17:42:11,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3794870-87a3-42b3-a4aa-de607dd0c74f" 2024-11-21T17:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3e60c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T17:42:11,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f3865e8353b4,46457,-1] 2024-11-21T17:42:11,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T17:42:11,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:42:11,764 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T17:42:11,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@704e0d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T17:42:11,769 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T17:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f3865e8353b4,46273,1732210898278, seqNum=-1] 2024-11-21T17:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T17:42:11,778 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T17:42:11,795 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f3865e8353b4,46457,1732210897685 2024-11-21T17:42:11,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd93155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T17:42:11,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f3865e8353b4,46457,-1 for getting cluster id 2024-11-21T17:42:11,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T17:42:11,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3794870-87a3-42b3-a4aa-de607dd0c74f' 2024-11-21T17:42:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T17:42:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3794870-87a3-42b3-a4aa-de607dd0c74f" 2024-11-21T17:42:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670089d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T17:42:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f3865e8353b4,46457,-1] 2024-11-21T17:42:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T17:42:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:42:11,802 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T17:42:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1950e714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T17:42:11,816 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=2353 2024-11-21T17:42:11,822 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T17:42:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f3865e8353b4,46457,1732210897685 2024-11-21T17:42:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33f43669 2024-11-21T17:42:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T17:42:11,833 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T17:42:11,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-21T17:42:11,849 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:11,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-21T17:42:11,852 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:11,854 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T17:42:11,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741835_1011 (size=428) 2024-11-21T17:42:12,277 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 573f91880d4a7d271704d294e32c1169, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:12,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741836_1012 (size=79) 2024-11-21T17:42:12,688 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:12,688 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 573f91880d4a7d271704d294e32c1169, disabling compactions & flushes 2024-11-21T17:42:12,688 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:42:12,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:42:12,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. after waiting 0 ms 2024-11-21T17:42:12,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:42:12,689 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:42:12,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 573f91880d4a7d271704d294e32c1169: Waiting for close lock at 1732210932688Disabling compacts and flushes for region at 1732210932688Disabling writes for close at 1732210932689 (+1 ms)Writing region close event to WAL at 1732210932689Closed at 1732210932689 2024-11-21T17:42:12,692 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:12,699 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732210932692"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210932692"}]},"ts":"1732210932692"} 2024-11-21T17:42:12,703 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:12,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:12,708 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210932705"}]},"ts":"1732210932705"} 2024-11-21T17:42:12,712 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-21T17:42:12,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=573f91880d4a7d271704d294e32c1169, ASSIGN}] 2024-11-21T17:42:12,717 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=573f91880d4a7d271704d294e32c1169, ASSIGN 2024-11-21T17:42:12,718 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=573f91880d4a7d271704d294e32c1169, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T17:42:12,872 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=573f91880d4a7d271704d294e32c1169, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:12,879 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=573f91880d4a7d271704d294e32c1169, ASSIGN because future has completed 2024-11-21T17:42:12,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573f91880d4a7d271704d294e32c1169, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:13,043 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:42:13,043 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 573f91880d4a7d271704d294e32c1169, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:13,044 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,044 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:13,044 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,044 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,047 INFO [StoreOpener-573f91880d4a7d271704d294e32c1169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,050 INFO [StoreOpener-573f91880d4a7d271704d294e32c1169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 573f91880d4a7d271704d294e32c1169 columnFamilyName testFamily 2024-11-21T17:42:13,050 DEBUG [StoreOpener-573f91880d4a7d271704d294e32c1169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:13,051 INFO [StoreOpener-573f91880d4a7d271704d294e32c1169-1 {}] regionserver.HStore(327): Store=573f91880d4a7d271704d294e32c1169/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:13,051 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,053 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,053 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,054 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,054 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,057 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,061 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:13,061 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 573f91880d4a7d271704d294e32c1169; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68013228, jitterRate=0.013476073741912842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:13,062 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:42:13,063 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 573f91880d4a7d271704d294e32c1169: Running coprocessor pre-open hook at 1732210933044Writing region info on filesystem at 1732210933044Initializing all the Stores at 1732210933047 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210933047Cleaning up temporary data from old regions at 1732210933054 (+7 ms)Running coprocessor post-open hooks at 1732210933062 (+8 ms)Region opened successfully at 1732210933062 2024-11-21T17:42:13,064 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169., pid=6, masterSystemTime=1732210933035 2024-11-21T17:42:13,068 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:42:13,068 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:42:13,069 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=573f91880d4a7d271704d294e32c1169, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:13,074 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 573f91880d4a7d271704d294e32c1169, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:13,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T17:42:13,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 573f91880d4a7d271704d294e32c1169, server=f3865e8353b4,46273,1732210898278 in 196 msec 2024-11-21T17:42:13,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T17:42:13,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=573f91880d4a7d271704d294e32c1169, ASSIGN in 366 msec 2024-11-21T17:42:13,086 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:13,086 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210933086"}]},"ts":"1732210933086"} 2024-11-21T17:42:13,089 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-21T17:42:13,090 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:13,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2490 sec 2024-11-21T17:42:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T17:42:14,897 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-21T17:42:14,897 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-21T17:42:14,899 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:14,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-21T17:42:14,907 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:14,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-21T17:42:14,910 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T17:42:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f3865e8353b4,46273,1732210898278, seqNum=-1] 2024-11-21T17:42:14,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T17:42:14,915 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T17:42:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:14,936 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:14,940 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-21T17:42:14,945 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-21T17:42:14,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-21T17:42:14,952 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-21T17:42:14,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5549814829876772872 2024-11-21T17:42:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210937952, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:14,980 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-21T17:42:14,982 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-21T17:42:14,982 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:14,984 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:14,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5549814829876772874 2024-11-21T17:42:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210937984, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:14,989 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:14,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=5549814829876772875 2024-11-21T17:42:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210937989, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:14,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:14,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=5549814829876772876 2024-11-21T17:42:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210937994, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:14,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:15,008 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1333031793_22 at /127.0.0.1:53948 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=187 (was 187), ProcessCount=11 (was 11), AvailableMemoryMB=2336 (was 2353) 2024-11-21T17:42:15,016 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=2335 2024-11-21T17:42:15,020 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:15,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-21T17:42:15,023 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:15,023 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:15,024 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-21T17:42:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T17:42:15,026 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:15,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741837_1013 (size=414) 2024-11-21T17:42:15,443 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4713e100e366f9b6cb42b459fb9993b3, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:15,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741838_1014 (size=65) 2024-11-21T17:42:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 4713e100e366f9b6cb42b459fb9993b3, disabling compactions & flushes 2024-11-21T17:42:15,856 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:42:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:42:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. after waiting 0 ms 2024-11-21T17:42:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:42:15,856 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:42:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4713e100e366f9b6cb42b459fb9993b3: Waiting for close lock at 1732210935856Disabling compacts and flushes for region at 1732210935856Disabling writes for close at 1732210935856Writing region close event to WAL at 1732210935856Closed at 1732210935856 2024-11-21T17:42:15,858 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:15,859 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732210935859"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210935859"}]},"ts":"1732210935859"} 2024-11-21T17:42:15,862 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:15,864 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:15,864 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210935864"}]},"ts":"1732210935864"} 2024-11-21T17:42:15,867 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-21T17:42:15,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4713e100e366f9b6cb42b459fb9993b3, ASSIGN}] 2024-11-21T17:42:15,870 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4713e100e366f9b6cb42b459fb9993b3, ASSIGN 2024-11-21T17:42:15,871 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4713e100e366f9b6cb42b459fb9993b3, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:16,022 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4713e100e366f9b6cb42b459fb9993b3, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:16,032 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4713e100e366f9b6cb42b459fb9993b3, ASSIGN because future has completed 2024-11-21T17:42:16,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4713e100e366f9b6cb42b459fb9993b3, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T17:42:16,200 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:42:16,200 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 4713e100e366f9b6cb42b459fb9993b3, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:16,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:16,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,205 INFO [StoreOpener-4713e100e366f9b6cb42b459fb9993b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,207 INFO [StoreOpener-4713e100e366f9b6cb42b459fb9993b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4713e100e366f9b6cb42b459fb9993b3 columnFamilyName testFamily 2024-11-21T17:42:16,207 DEBUG [StoreOpener-4713e100e366f9b6cb42b459fb9993b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:16,207 INFO [StoreOpener-4713e100e366f9b6cb42b459fb9993b3-1 {}] regionserver.HStore(327): Store=4713e100e366f9b6cb42b459fb9993b3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:16,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,209 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,209 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,210 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,210 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,213 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:16,217 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 4713e100e366f9b6cb42b459fb9993b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61390579, jitterRate=-0.08520908653736115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:16,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:42:16,218 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 4713e100e366f9b6cb42b459fb9993b3: Running coprocessor pre-open hook at 1732210936202Writing region info on filesystem at 1732210936202Initializing all the Stores at 1732210936204 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210936204Cleaning up temporary data from old regions at 1732210936210 (+6 ms)Running coprocessor post-open hooks at 1732210936217 (+7 ms)Region opened successfully at 1732210936218 (+1 ms) 2024-11-21T17:42:16,219 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3., pid=9, masterSystemTime=1732210936189 2024-11-21T17:42:16,222 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:42:16,222 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:42:16,223 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4713e100e366f9b6cb42b459fb9993b3, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:16,227 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4713e100e366f9b6cb42b459fb9993b3, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:16,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-21T17:42:16,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 4713e100e366f9b6cb42b459fb9993b3, server=f3865e8353b4,46273,1732210898278 in 194 msec 2024-11-21T17:42:16,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T17:42:16,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=4713e100e366f9b6cb42b459fb9993b3, ASSIGN in 364 msec 2024-11-21T17:42:16,237 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:16,238 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210936237"}]},"ts":"1732210936237"} 2024-11-21T17:42:16,241 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-21T17:42:16,243 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:16,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2230 sec 2024-11-21T17:42:17,026 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:42:17,051 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-21T17:42:17,052 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T17:42:17,053 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-21T17:42:18,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T17:42:18,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:18,039 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T17:42:18,039 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T17:42:18,040 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-21T17:42:18,040 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:18,044 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-21T17:42:18,044 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T17:42:18,053 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-21T17:42:18,053 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-21T17:42:18,053 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:18,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-21T17:42:18,058 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:18,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-21T17:42:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:18,073 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:18,074 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-21T17:42:18,078 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-21T17:42:18,080 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-21T17:42:24,099 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3., hostname=f3865e8353b4,46273,1732210898278, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f3865e8353b4:46273 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6018ms, rpcTimeout=5999ms 2024-11-21T17:42:24,100 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6018ms, rpcTimeout=5999ms 2024-11-21T17:42:24,100 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 from cache 2024-11-21T17:42:24,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=165 (was 187), ProcessCount=11 (was 11), AvailableMemoryMB=2313 (was 2335) 2024-11-21T17:42:24,119 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=2313 2024-11-21T17:42:24,122 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:24,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-21T17:42:24,126 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:24,126 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:24,126 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-21T17:42:24,128 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:24,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-21T17:42:24,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741839_1015 (size=412) 2024-11-21T17:42:24,538 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 174ea425cc45b438f69b2ae95208b995, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:24,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741840_1016 (size=63) 2024-11-21T17:42:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-21T17:42:24,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:24,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 174ea425cc45b438f69b2ae95208b995, disabling compactions & flushes 2024-11-21T17:42:24,951 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:42:24,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:42:24,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. after waiting 0 ms 2024-11-21T17:42:24,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:42:24,952 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:42:24,952 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 174ea425cc45b438f69b2ae95208b995: Waiting for close lock at 1732210944951Disabling compacts and flushes for region at 1732210944951Disabling writes for close at 1732210944952 (+1 ms)Writing region close event to WAL at 1732210944952Closed at 1732210944952 2024-11-21T17:42:24,958 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:24,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732210944958"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210944958"}]},"ts":"1732210944958"} 2024-11-21T17:42:24,962 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:24,964 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:24,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210944964"}]},"ts":"1732210944964"} 2024-11-21T17:42:24,968 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-21T17:42:24,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=174ea425cc45b438f69b2ae95208b995, ASSIGN}] 2024-11-21T17:42:24,971 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=174ea425cc45b438f69b2ae95208b995, ASSIGN 2024-11-21T17:42:24,973 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=174ea425cc45b438f69b2ae95208b995, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:25,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=174ea425cc45b438f69b2ae95208b995, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:25,134 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=174ea425cc45b438f69b2ae95208b995, ASSIGN because future has completed 2024-11-21T17:42:25,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 174ea425cc45b438f69b2ae95208b995, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-21T17:42:25,301 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:42:25,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 174ea425cc45b438f69b2ae95208b995, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:25,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:25,303 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,303 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,306 INFO [StoreOpener-174ea425cc45b438f69b2ae95208b995-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,309 INFO [StoreOpener-174ea425cc45b438f69b2ae95208b995-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 174ea425cc45b438f69b2ae95208b995 columnFamilyName testFamily 2024-11-21T17:42:25,309 DEBUG [StoreOpener-174ea425cc45b438f69b2ae95208b995-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:25,310 INFO [StoreOpener-174ea425cc45b438f69b2ae95208b995-1 {}] regionserver.HStore(327): Store=174ea425cc45b438f69b2ae95208b995/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:25,310 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,312 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,313 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,313 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,313 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,319 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:25,320 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 174ea425cc45b438f69b2ae95208b995; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69177120, jitterRate=0.030819416046142578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:25,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:42:25,321 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 174ea425cc45b438f69b2ae95208b995: Running coprocessor pre-open hook at 1732210945303Writing region info on filesystem at 1732210945303Initializing all the Stores at 1732210945305 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210945305Cleaning up temporary data from old regions at 1732210945313 (+8 ms)Running coprocessor post-open hooks at 1732210945320 (+7 ms)Region opened successfully at 1732210945321 (+1 ms) 2024-11-21T17:42:25,323 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995., pid=12, masterSystemTime=1732210945289 2024-11-21T17:42:25,326 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:42:25,326 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:42:25,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=174ea425cc45b438f69b2ae95208b995, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:25,330 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 174ea425cc45b438f69b2ae95208b995, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:25,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T17:42:25,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 174ea425cc45b438f69b2ae95208b995, server=f3865e8353b4,46273,1732210898278 in 198 msec 2024-11-21T17:42:25,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-21T17:42:25,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=174ea425cc45b438f69b2ae95208b995, ASSIGN in 367 msec 2024-11-21T17:42:25,340 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:25,340 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210945340"}]},"ts":"1732210945340"} 2024-11-21T17:42:25,343 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-21T17:42:25,345 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:25,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2230 sec 2024-11-21T17:42:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-21T17:42:27,175 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-21T17:42:27,175 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-21T17:42:27,176 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:27,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-21T17:42:27,185 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:27,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-21T17:42:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:27,201 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:27,201 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-21T17:42:27,205 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-21T17:42:27,207 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-21T17:42:27,209 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-21T17:42:27,209 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-21T17:42:28,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-21T17:42:28,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:29,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-21T17:42:30,844 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:42:33,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46273 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772885' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-21T17:42:33,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46273 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43374 deadline: 1732210959220, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772885' does not exist 2024-11-21T17:42:33,232 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f3865e8353b4,46273,1732210898278 for closing scanner id = 5549814829876772885 for 174ea425cc45b438f69b2ae95208b995 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772885' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T17:42:33,235 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=156 (was 165), ProcessCount=11 (was 11), AvailableMemoryMB=2291 (was 2313) 2024-11-21T17:42:33,244 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=156, ProcessCount=11, AvailableMemoryMB=2290 2024-11-21T17:42:33,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-21T17:42:33,250 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:33,251 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:33,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-21T17:42:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T17:42:33,252 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:33,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741841_1017 (size=423) 2024-11-21T17:42:33,670 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d43585d0894151ec1d56b74d358e2b3b, NAME => 'testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:33,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741842_1018 (size=74) 2024-11-21T17:42:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-21T17:42:34,087 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:34,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing d43585d0894151ec1d56b74d358e2b3b, disabling compactions & flushes 2024-11-21T17:42:34,088 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:42:34,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:42:34,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. after waiting 0 ms 2024-11-21T17:42:34,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:42:34,089 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:42:34,089 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for d43585d0894151ec1d56b74d358e2b3b: Waiting for close lock at 1732210954087Disabling compacts and flushes for region at 1732210954087Disabling writes for close at 1732210954088 (+1 ms)Writing region close event to WAL at 1732210954088Closed at 1732210954088 2024-11-21T17:42:34,093 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:34,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732210954093"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210954093"}]},"ts":"1732210954093"} 2024-11-21T17:42:34,097 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:34,099 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:34,100 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210954100"}]},"ts":"1732210954100"} 2024-11-21T17:42:34,104 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-21T17:42:34,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d43585d0894151ec1d56b74d358e2b3b, ASSIGN}] 2024-11-21T17:42:34,107 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d43585d0894151ec1d56b74d358e2b3b, ASSIGN 2024-11-21T17:42:34,109 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d43585d0894151ec1d56b74d358e2b3b, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:34,261 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d43585d0894151ec1d56b74d358e2b3b, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T17:42:34,269 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d43585d0894151ec1d56b74d358e2b3b, ASSIGN because future has completed 2024-11-21T17:42:34,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d43585d0894151ec1d56b74d358e2b3b, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:34,438 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:42:34,438 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => d43585d0894151ec1d56b74d358e2b3b, NAME => 'testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:34,439 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,439 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:34,439 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,439 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,442 INFO [StoreOpener-d43585d0894151ec1d56b74d358e2b3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,446 INFO [StoreOpener-d43585d0894151ec1d56b74d358e2b3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d43585d0894151ec1d56b74d358e2b3b columnFamilyName testFamily 2024-11-21T17:42:34,447 DEBUG [StoreOpener-d43585d0894151ec1d56b74d358e2b3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:34,448 INFO [StoreOpener-d43585d0894151ec1d56b74d358e2b3b-1 {}] regionserver.HStore(327): Store=d43585d0894151ec1d56b74d358e2b3b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:34,448 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,450 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,451 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,452 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,452 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,455 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,460 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:34,461 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened d43585d0894151ec1d56b74d358e2b3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62870385, jitterRate=-0.06315825879573822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:34,462 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:42:34,463 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for d43585d0894151ec1d56b74d358e2b3b: Running coprocessor pre-open hook at 1732210954439Writing region info on filesystem at 1732210954439Initializing all the Stores at 1732210954441 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210954442 (+1 ms)Cleaning up temporary data from old regions at 1732210954452 (+10 ms)Running coprocessor post-open hooks at 1732210954462 (+10 ms)Region opened successfully at 1732210954462 2024-11-21T17:42:34,464 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b., pid=15, masterSystemTime=1732210954425 2024-11-21T17:42:34,467 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:42:34,467 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:42:34,468 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d43585d0894151ec1d56b74d358e2b3b, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:34,471 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d43585d0894151ec1d56b74d358e2b3b, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:34,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-21T17:42:34,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure d43585d0894151ec1d56b74d358e2b3b, server=f3865e8353b4,46273,1732210898278 in 202 msec 2024-11-21T17:42:34,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T17:42:34,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d43585d0894151ec1d56b74d358e2b3b, ASSIGN in 373 msec 2024-11-21T17:42:34,482 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:34,482 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210954482"}]},"ts":"1732210954482"} 2024-11-21T17:42:34,485 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-21T17:42:34,486 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:34,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2390 sec 2024-11-21T17:42:35,059 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-21T17:42:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T17:42:36,294 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-21T17:42:36,294 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-21T17:42:36,295 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:36,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-21T17:42:36,304 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:36,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-21T17:42:36,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:36,320 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:36,320 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-21T17:42:36,323 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-21T17:42:36,325 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-21T17:42:36,326 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-21T17:42:36,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5549814829876772889 2024-11-21T17:42:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210959325, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:36,329 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-21T17:42:36,331 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-21T17:42:36,331 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:36,333 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:36,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5549814829876772891 2024-11-21T17:42:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210959332, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:36,336 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:36,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=5549814829876772892 2024-11-21T17:42:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210959335, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:36,339 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:36,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=5549814829876772893 2024-11-21T17:42:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43374 deadline: 1732210959339, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-21T17:42:36,342 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-21T17:42:36,350 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1333031793_22 at /127.0.0.1:45504 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 156), ProcessCount=11 (was 11), AvailableMemoryMB=2284 (was 2290) 2024-11-21T17:42:36,358 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=2284 2024-11-21T17:42:36,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-21T17:42:36,365 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:36,365 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:36,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-21T17:42:36,367 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-21T17:42:36,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741843_1019 (size=418) 2024-11-21T17:42:36,786 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d49dabad34469f3e689a915ad669f6d4, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:36,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741844_1020 (size=69) 2024-11-21T17:42:36,849 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T17:42:37,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:37,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d49dabad34469f3e689a915ad669f6d4, disabling compactions & flushes 2024-11-21T17:42:37,200 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:42:37,200 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:42:37,200 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. after waiting 0 ms 2024-11-21T17:42:37,200 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:42:37,200 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:42:37,201 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d49dabad34469f3e689a915ad669f6d4: Waiting for close lock at 1732210957199Disabling compacts and flushes for region at 1732210957199Disabling writes for close at 1732210957200 (+1 ms)Writing region close event to WAL at 1732210957200Closed at 1732210957200 2024-11-21T17:42:37,206 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:37,207 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732210957206"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210957206"}]},"ts":"1732210957206"} 2024-11-21T17:42:37,211 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:37,213 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:37,213 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210957213"}]},"ts":"1732210957213"} 2024-11-21T17:42:37,216 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-21T17:42:37,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d49dabad34469f3e689a915ad669f6d4, ASSIGN}] 2024-11-21T17:42:37,219 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d49dabad34469f3e689a915ad669f6d4, ASSIGN 2024-11-21T17:42:37,220 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d49dabad34469f3e689a915ad669f6d4, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:37,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d49dabad34469f3e689a915ad669f6d4, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:37,378 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d49dabad34469f3e689a915ad669f6d4, ASSIGN because future has completed 2024-11-21T17:42:37,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d49dabad34469f3e689a915ad669f6d4, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-21T17:42:37,545 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:42:37,545 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => d49dabad34469f3e689a915ad669f6d4, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:37,546 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,546 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:37,546 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,546 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,548 INFO [StoreOpener-d49dabad34469f3e689a915ad669f6d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,550 INFO [StoreOpener-d49dabad34469f3e689a915ad669f6d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49dabad34469f3e689a915ad669f6d4 columnFamilyName testFamily 2024-11-21T17:42:37,550 DEBUG [StoreOpener-d49dabad34469f3e689a915ad669f6d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:37,551 INFO [StoreOpener-d49dabad34469f3e689a915ad669f6d4-1 {}] regionserver.HStore(327): Store=d49dabad34469f3e689a915ad669f6d4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:37,551 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,552 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,552 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,553 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,553 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,555 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,558 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:37,559 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened d49dabad34469f3e689a915ad669f6d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59317514, jitterRate=-0.11610016226768494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:37,559 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:42:37,560 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for d49dabad34469f3e689a915ad669f6d4: Running coprocessor pre-open hook at 1732210957546Writing region info on filesystem at 1732210957546Initializing all the Stores at 1732210957548 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210957548Cleaning up temporary data from old regions at 1732210957553 (+5 ms)Running coprocessor post-open hooks at 1732210957559 (+6 ms)Region opened successfully at 1732210957560 (+1 ms) 2024-11-21T17:42:37,561 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., pid=18, masterSystemTime=1732210957535 2024-11-21T17:42:37,564 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:42:37,564 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:42:37,564 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d49dabad34469f3e689a915ad669f6d4, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:37,567 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d49dabad34469f3e689a915ad669f6d4, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:37,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T17:42:37,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d49dabad34469f3e689a915ad669f6d4, server=f3865e8353b4,46273,1732210898278 in 189 msec 2024-11-21T17:42:37,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-21T17:42:37,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d49dabad34469f3e689a915ad669f6d4, ASSIGN in 354 msec 2024-11-21T17:42:37,575 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:37,575 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210957575"}]},"ts":"1732210957575"} 2024-11-21T17:42:37,578 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-21T17:42:37,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:37,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2180 sec 2024-11-21T17:42:38,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-21T17:42:38,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:38,038 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-21T17:42:38,038 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-21T17:42:39,405 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-21T17:42:39,405 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-21T17:42:39,405 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:39,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-21T17:42:39,415 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:39,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-21T17:42:39,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:39,428 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:39,428 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-21T17:42:39,431 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-21T17:42:39,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-21T17:42:39,943 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:42:40,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f3865e8353b4:46273 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-21T17:42:40,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-21T17:42:40,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 from cache 2024-11-21T17:42:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-21T17:42:41,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-21T17:42:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:41,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-21T17:42:42,438 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f3865e8353b4:46273 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=981ms, rpcTimeout=974ms 2024-11-21T17:42:42,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=981ms, rpcTimeout=974ms 2024-11-21T17:42:42,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 from cache 2024-11-21T17:42:42,452 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 226), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 143), ProcessCount=11 (was 11), AvailableMemoryMB=2278 (was 2284) 2024-11-21T17:42:42,460 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=2278 2024-11-21T17:42:42,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-21T17:42:42,466 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:42,466 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:42,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-21T17:42:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T17:42:42,468 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:42,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741845_1021 (size=416) 2024-11-21T17:42:42,881 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c20801e25e613900913d41665a005d9c, NAME => 'testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:42,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741846_1022 (size=67) 2024-11-21T17:42:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-21T17:42:43,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:43,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing c20801e25e613900913d41665a005d9c, disabling compactions & flushes 2024-11-21T17:42:43,297 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:42:43,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:42:43,298 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. after waiting 0 ms 2024-11-21T17:42:43,298 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:42:43,298 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:42:43,299 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for c20801e25e613900913d41665a005d9c: Waiting for close lock at 1732210963297Disabling compacts and flushes for region at 1732210963297Disabling writes for close at 1732210963298 (+1 ms)Writing region close event to WAL at 1732210963298Closed at 1732210963298 2024-11-21T17:42:43,304 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:43,305 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732210963304"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210963304"}]},"ts":"1732210963304"} 2024-11-21T17:42:43,308 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:43,309 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:43,310 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210963310"}]},"ts":"1732210963310"} 2024-11-21T17:42:43,312 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-21T17:42:43,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=c20801e25e613900913d41665a005d9c, ASSIGN}] 2024-11-21T17:42:43,315 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=c20801e25e613900913d41665a005d9c, ASSIGN 2024-11-21T17:42:43,316 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=c20801e25e613900913d41665a005d9c, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:43,468 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c20801e25e613900913d41665a005d9c, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:43,473 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=c20801e25e613900913d41665a005d9c, ASSIGN because future has completed 2024-11-21T17:42:43,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c20801e25e613900913d41665a005d9c, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:43,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T17:42:43,640 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:42:43,640 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => c20801e25e613900913d41665a005d9c, NAME => 'testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:43,640 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,641 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:43,641 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,641 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,643 INFO [StoreOpener-c20801e25e613900913d41665a005d9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,645 INFO [StoreOpener-c20801e25e613900913d41665a005d9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c20801e25e613900913d41665a005d9c columnFamilyName testFamily 2024-11-21T17:42:43,645 DEBUG [StoreOpener-c20801e25e613900913d41665a005d9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:43,645 INFO [StoreOpener-c20801e25e613900913d41665a005d9c-1 {}] regionserver.HStore(327): Store=c20801e25e613900913d41665a005d9c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:43,645 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,647 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,647 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,648 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,648 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,650 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,653 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:43,653 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened c20801e25e613900913d41665a005d9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64546206, jitterRate=-0.03818657994270325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:43,654 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c20801e25e613900913d41665a005d9c 2024-11-21T17:42:43,654 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for c20801e25e613900913d41665a005d9c: Running coprocessor pre-open hook at 1732210963641Writing region info on filesystem at 1732210963641Initializing all the Stores at 1732210963642 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210963642Cleaning up temporary data from old regions at 1732210963648 (+6 ms)Running coprocessor post-open hooks at 1732210963654 (+6 ms)Region opened successfully at 1732210963654 2024-11-21T17:42:43,655 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c., pid=21, masterSystemTime=1732210963628 2024-11-21T17:42:43,658 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:42:43,658 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:42:43,659 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c20801e25e613900913d41665a005d9c, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:43,662 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c20801e25e613900913d41665a005d9c, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:43,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-21T17:42:43,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure c20801e25e613900913d41665a005d9c, server=f3865e8353b4,46273,1732210898278 in 191 msec 2024-11-21T17:42:43,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T17:42:43,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=c20801e25e613900913d41665a005d9c, ASSIGN in 354 msec 2024-11-21T17:42:43,671 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:43,671 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210963671"}]},"ts":"1732210963671"} 2024-11-21T17:42:43,673 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-21T17:42:43,674 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:43,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2120 sec 2024-11-21T17:42:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T17:42:45,516 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-21T17:42:45,516 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-21T17:42:45,516 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:45,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-21T17:42:45,528 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:45,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-21T17:42:45,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:45,541 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:45,542 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-21T17:42:45,545 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-21T17:42:45,548 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-21T17:42:45,550 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-21T17:42:45,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-21T17:42:47,064 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-21T17:42:48,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-21T17:42:48,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:48,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46273 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772904' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-21T17:42:48,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46273 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43374 deadline: 1732210969565, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772904' does not exist 2024-11-21T17:42:48,567 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f3865e8353b4,46273,1732210898278 for closing scanner id = 5549814829876772904 for c20801e25e613900913d41665a005d9c of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772904' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T17:42:48,574 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=128 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=2268 (was 2278) 2024-11-21T17:42:48,584 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=128, ProcessCount=11, AvailableMemoryMB=2268 2024-11-21T17:42:48,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-21T17:42:48,591 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:48,591 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:48,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-21T17:42:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-21T17:42:48,594 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:48,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741847_1023 (size=411) 2024-11-21T17:42:49,008 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 78559f7b90316348fa9afb96424506ab, NAME => 'testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:49,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741848_1024 (size=62) 2024-11-21T17:42:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-21T17:42:49,145 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:42:49,422 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 78559f7b90316348fa9afb96424506ab, disabling compactions & flushes 2024-11-21T17:42:49,423 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:42:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:42:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. after waiting 0 ms 2024-11-21T17:42:49,423 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:42:49,424 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:42:49,424 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 78559f7b90316348fa9afb96424506ab: Waiting for close lock at 1732210969423Disabling compacts and flushes for region at 1732210969423Disabling writes for close at 1732210969423Writing region close event to WAL at 1732210969423Closed at 1732210969423 2024-11-21T17:42:49,426 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:49,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732210969426"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210969426"}]},"ts":"1732210969426"} 2024-11-21T17:42:49,430 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:49,432 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:49,433 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210969432"}]},"ts":"1732210969432"} 2024-11-21T17:42:49,436 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-21T17:42:49,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=78559f7b90316348fa9afb96424506ab, ASSIGN}] 2024-11-21T17:42:49,439 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=78559f7b90316348fa9afb96424506ab, ASSIGN 2024-11-21T17:42:49,441 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=78559f7b90316348fa9afb96424506ab, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:49,592 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=78559f7b90316348fa9afb96424506ab, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:49,602 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=78559f7b90316348fa9afb96424506ab, ASSIGN because future has completed 2024-11-21T17:42:49,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78559f7b90316348fa9afb96424506ab, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-21T17:42:49,769 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:42:49,769 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 78559f7b90316348fa9afb96424506ab, NAME => 'testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:49,770 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,770 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:49,771 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,771 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,774 INFO [StoreOpener-78559f7b90316348fa9afb96424506ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,777 INFO [StoreOpener-78559f7b90316348fa9afb96424506ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78559f7b90316348fa9afb96424506ab columnFamilyName testFamily 2024-11-21T17:42:49,777 DEBUG [StoreOpener-78559f7b90316348fa9afb96424506ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:49,778 INFO [StoreOpener-78559f7b90316348fa9afb96424506ab-1 {}] regionserver.HStore(327): Store=78559f7b90316348fa9afb96424506ab/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:49,778 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,779 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,780 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,781 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,781 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,784 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,789 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:49,790 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 78559f7b90316348fa9afb96424506ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67836537, jitterRate=0.010843172669410706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:49,790 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:42:49,791 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 78559f7b90316348fa9afb96424506ab: Running coprocessor pre-open hook at 1732210969771Writing region info on filesystem at 1732210969771Initializing all the Stores at 1732210969773 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210969773Cleaning up temporary data from old regions at 1732210969781 (+8 ms)Running coprocessor post-open hooks at 1732210969790 (+9 ms)Region opened successfully at 1732210969791 (+1 ms) 2024-11-21T17:42:49,792 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab., pid=24, masterSystemTime=1732210969757 2024-11-21T17:42:49,794 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:42:49,795 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:42:49,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=78559f7b90316348fa9afb96424506ab, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:49,798 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78559f7b90316348fa9afb96424506ab, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:49,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T17:42:49,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 78559f7b90316348fa9afb96424506ab, server=f3865e8353b4,46273,1732210898278 in 197 msec 2024-11-21T17:42:49,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-21T17:42:49,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=78559f7b90316348fa9afb96424506ab, ASSIGN in 365 msec 2024-11-21T17:42:49,805 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:49,805 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210969805"}]},"ts":"1732210969805"} 2024-11-21T17:42:49,808 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-21T17:42:49,809 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:49,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2220 sec 2024-11-21T17:42:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-21T17:42:51,636 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-21T17:42:51,636 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-21T17:42:51,636 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:51,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-21T17:42:51,647 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:51,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-21T17:42:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:51,660 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:51,660 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-21T17:42:51,664 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-21T17:42:51,667 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-21T17:42:51,669 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-21T17:42:51,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-21T17:42:52,464 INFO [master/f3865e8353b4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T17:42:52,464 INFO [master/f3865e8353b4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T17:42:53,068 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-21T17:42:54,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46273 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772908' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-21T17:42:54,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46273 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43374 deadline: 1732210975685, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772908' does not exist 2024-11-21T17:42:54,687 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f3865e8353b4,46273,1732210898278 for closing scanner id = 5549814829876772908 for 78559f7b90316348fa9afb96424506ab of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772908' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T17:42:54,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=134 (was 128) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2256 (was 2268) 2024-11-21T17:42:54,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=134, ProcessCount=11, AvailableMemoryMB=2255 2024-11-21T17:42:54,708 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:42:54,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-21T17:42:54,712 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:42:54,712 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:54,712 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-21T17:42:54,714 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:42:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T17:42:54,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741849_1025 (size=419) 2024-11-21T17:42:55,130 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6980b612a9341bf9afba0c7914664b14, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:42:55,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741850_1026 (size=70) 2024-11-21T17:42:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-21T17:42:55,275 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:42:55,542 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:55,542 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6980b612a9341bf9afba0c7914664b14, disabling compactions & flushes 2024-11-21T17:42:55,542 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:42:55,543 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:42:55,543 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. after waiting 0 ms 2024-11-21T17:42:55,543 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:42:55,543 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:42:55,543 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6980b612a9341bf9afba0c7914664b14: Waiting for close lock at 1732210975542Disabling compacts and flushes for region at 1732210975542Disabling writes for close at 1732210975543 (+1 ms)Writing region close event to WAL at 1732210975543Closed at 1732210975543 2024-11-21T17:42:55,546 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:42:55,546 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732210975546"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210975546"}]},"ts":"1732210975546"} 2024-11-21T17:42:55,550 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:42:55,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:42:55,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210975552"}]},"ts":"1732210975552"} 2024-11-21T17:42:55,556 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-21T17:42:55,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=6980b612a9341bf9afba0c7914664b14, ASSIGN}] 2024-11-21T17:42:55,558 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=6980b612a9341bf9afba0c7914664b14, ASSIGN 2024-11-21T17:42:55,559 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=6980b612a9341bf9afba0c7914664b14, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:42:55,711 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=6980b612a9341bf9afba0c7914664b14, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:55,720 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=6980b612a9341bf9afba0c7914664b14, ASSIGN because future has completed 2024-11-21T17:42:55,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6980b612a9341bf9afba0c7914664b14, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:42:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T17:42:55,886 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:42:55,887 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 6980b612a9341bf9afba0c7914664b14, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:42:55,887 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,887 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:42:55,888 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,888 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,891 INFO [StoreOpener-6980b612a9341bf9afba0c7914664b14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,893 INFO [StoreOpener-6980b612a9341bf9afba0c7914664b14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6980b612a9341bf9afba0c7914664b14 columnFamilyName testFamily 2024-11-21T17:42:55,894 DEBUG [StoreOpener-6980b612a9341bf9afba0c7914664b14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:42:55,894 INFO [StoreOpener-6980b612a9341bf9afba0c7914664b14-1 {}] regionserver.HStore(327): Store=6980b612a9341bf9afba0c7914664b14/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:42:55,895 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,896 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,898 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,898 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,901 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,905 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:42:55,906 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 6980b612a9341bf9afba0c7914664b14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71719224, jitterRate=0.06869971752166748}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:42:55,907 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:42:55,907 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 6980b612a9341bf9afba0c7914664b14: Running coprocessor pre-open hook at 1732210975888Writing region info on filesystem at 1732210975888Initializing all the Stores at 1732210975890 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210975890Cleaning up temporary data from old regions at 1732210975898 (+8 ms)Running coprocessor post-open hooks at 1732210975907 (+9 ms)Region opened successfully at 1732210975907 2024-11-21T17:42:55,909 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., pid=27, masterSystemTime=1732210975874 2024-11-21T17:42:55,911 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:42:55,911 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:42:55,912 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=6980b612a9341bf9afba0c7914664b14, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:42:55,914 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6980b612a9341bf9afba0c7914664b14, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:42:55,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-21T17:42:55,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 6980b612a9341bf9afba0c7914664b14, server=f3865e8353b4,46273,1732210898278 in 196 msec 2024-11-21T17:42:55,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T17:42:55,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=6980b612a9341bf9afba0c7914664b14, ASSIGN in 362 msec 2024-11-21T17:42:55,922 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:42:55,922 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210975922"}]},"ts":"1732210975922"} 2024-11-21T17:42:55,924 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-21T17:42:55,925 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:42:55,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2170 sec 2024-11-21T17:42:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T17:42:57,745 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-21T17:42:57,745 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-21T17:42:57,746 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:57,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-21T17:42:57,755 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:42:57,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-21T17:42:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:42:57,768 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:42:57,769 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-21T17:42:57,773 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-21T17:42:57,775 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-21T17:42:58,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-21T17:42:58,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:58,036 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-21T17:42:58,036 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-21T17:42:59,072 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-21T17:43:01,392 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:43:03,779 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., hostname=f3865e8353b4,46273,1732210898278, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f3865e8353b4:46273 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-21T17:43:03,779 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-21T17:43:03,779 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 from cache 2024-11-21T17:43:03,794 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=113 (was 134), ProcessCount=11 (was 11), AvailableMemoryMB=2239 (was 2255) 2024-11-21T17:43:03,803 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=113, ProcessCount=11, AvailableMemoryMB=2239 2024-11-21T17:43:03,806 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:43:03,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-21T17:43:03,810 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:43:03,810 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:43:03,810 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-21T17:43:03,811 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:43:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T17:43:03,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741851_1027 (size=407) 2024-11-21T17:43:04,229 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 27ef19f9c166277483f3daf183d8a4da, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:43:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741852_1028 (size=58) 2024-11-21T17:43:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-21T17:43:04,642 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:43:04,642 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 27ef19f9c166277483f3daf183d8a4da, disabling compactions & flushes 2024-11-21T17:43:04,642 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:04,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:04,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. after waiting 0 ms 2024-11-21T17:43:04,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:04,643 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:04,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 27ef19f9c166277483f3daf183d8a4da: Waiting for close lock at 1732210984642Disabling compacts and flushes for region at 1732210984642Disabling writes for close at 1732210984643 (+1 ms)Writing region close event to WAL at 1732210984643Closed at 1732210984643 2024-11-21T17:43:04,648 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:43:04,649 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732210984648"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210984648"}]},"ts":"1732210984648"} 2024-11-21T17:43:04,654 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:43:04,655 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:43:04,655 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210984655"}]},"ts":"1732210984655"} 2024-11-21T17:43:04,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-21T17:43:04,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=27ef19f9c166277483f3daf183d8a4da, ASSIGN}] 2024-11-21T17:43:04,659 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=27ef19f9c166277483f3daf183d8a4da, ASSIGN 2024-11-21T17:43:04,660 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=27ef19f9c166277483f3daf183d8a4da, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:43:04,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=27ef19f9c166277483f3daf183d8a4da, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:43:04,816 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=27ef19f9c166277483f3daf183d8a4da, ASSIGN because future has completed 2024-11-21T17:43:04,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27ef19f9c166277483f3daf183d8a4da, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:43:04,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T17:43:04,985 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:04,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 27ef19f9c166277483f3daf183d8a4da, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:43:04,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:43:04,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,988 INFO [StoreOpener-27ef19f9c166277483f3daf183d8a4da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,990 INFO [StoreOpener-27ef19f9c166277483f3daf183d8a4da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27ef19f9c166277483f3daf183d8a4da columnFamilyName testFamily 2024-11-21T17:43:04,990 DEBUG [StoreOpener-27ef19f9c166277483f3daf183d8a4da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:43:04,991 INFO [StoreOpener-27ef19f9c166277483f3daf183d8a4da-1 {}] regionserver.HStore(327): Store=27ef19f9c166277483f3daf183d8a4da/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:43:04,991 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,993 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,993 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,995 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:43:04,998 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 27ef19f9c166277483f3daf183d8a4da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59473982, jitterRate=-0.11376860737800598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:43:04,998 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:04,999 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 27ef19f9c166277483f3daf183d8a4da: Running coprocessor pre-open hook at 1732210984987Writing region info on filesystem at 1732210984987Initializing all the Stores at 1732210984988 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210984988Cleaning up temporary data from old regions at 1732210984993 (+5 ms)Running coprocessor post-open hooks at 1732210984998 (+5 ms)Region opened successfully at 1732210984999 (+1 ms) 2024-11-21T17:43:05,000 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da., pid=30, masterSystemTime=1732210984972 2024-11-21T17:43:05,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:05,003 INFO [RS_OPEN_PRIORITY_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:05,004 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=27ef19f9c166277483f3daf183d8a4da, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:43:05,006 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27ef19f9c166277483f3daf183d8a4da, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:43:05,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-21T17:43:05,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 27ef19f9c166277483f3daf183d8a4da, server=f3865e8353b4,46273,1732210898278 in 191 msec 2024-11-21T17:43:05,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-21T17:43:05,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=27ef19f9c166277483f3daf183d8a4da, ASSIGN in 353 msec 2024-11-21T17:43:05,013 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:43:05,014 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210985013"}]},"ts":"1732210985013"} 2024-11-21T17:43:05,016 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-21T17:43:05,016 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:43:05,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2110 sec 2024-11-21T17:43:05,076 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-21T17:43:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-21T17:43:06,846 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-21T17:43:06,846 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-21T17:43:06,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:43:06,849 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T17:43:06,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-21T17:43:06,857 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:43:06,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-21T17:43:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:43:06,872 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:43:06,872 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-21T17:43:06,875 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-21T17:43:06,877 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-21T17:43:06,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-21T17:43:06,879 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-21T17:43:08,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-21T17:43:08,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-21T17:43:10,493 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:43:11,652 DEBUG [master/f3865e8353b4:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T17:43:12,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46273 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772916' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-21T17:43:12,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46273 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43374 deadline: 1732210998895, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772916' does not exist 2024-11-21T17:43:12,899 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to f3865e8353b4,46273,1732210898278 for closing scanner id = 5549814829876772916 for 27ef19f9c166277483f3daf183d8a4da of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5549814829876772916' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T17:43:12,906 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=104 (was 113), ProcessCount=11 (was 11), AvailableMemoryMB=2216 (was 2239) 2024-11-21T17:43:12,914 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=104, ProcessCount=11, AvailableMemoryMB=2216 2024-11-21T17:43:12,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T17:43:12,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-21T17:43:12,920 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T17:43:12,920 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:43:12,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-21T17:43:12,921 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T17:43:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T17:43:12,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741853_1029 (size=423) 2024-11-21T17:43:13,335 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cd2f9a466065dd286ae3c9fbcf067da2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808 2024-11-21T17:43:13,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741854_1030 (size=74) 2024-11-21T17:43:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-21T17:43:13,748 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:43:13,749 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing cd2f9a466065dd286ae3c9fbcf067da2, disabling compactions & flushes 2024-11-21T17:43:13,749 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:13,749 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:13,749 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. after waiting 0 ms 2024-11-21T17:43:13,749 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:13,749 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:13,750 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for cd2f9a466065dd286ae3c9fbcf067da2: Waiting for close lock at 1732210993749Disabling compacts and flushes for region at 1732210993749Disabling writes for close at 1732210993749Writing region close event to WAL at 1732210993749Closed at 1732210993749 2024-11-21T17:43:13,753 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T17:43:13,753 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732210993753"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732210993753"}]},"ts":"1732210993753"} 2024-11-21T17:43:13,758 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T17:43:13,759 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T17:43:13,759 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210993759"}]},"ts":"1732210993759"} 2024-11-21T17:43:13,761 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-21T17:43:13,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cd2f9a466065dd286ae3c9fbcf067da2, ASSIGN}] 2024-11-21T17:43:13,763 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cd2f9a466065dd286ae3c9fbcf067da2, ASSIGN 2024-11-21T17:43:13,764 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cd2f9a466065dd286ae3c9fbcf067da2, ASSIGN; state=OFFLINE, location=f3865e8353b4,46273,1732210898278; forceNewPlan=false, retain=false 2024-11-21T17:43:13,915 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=cd2f9a466065dd286ae3c9fbcf067da2, regionState=OPENING, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:43:13,919 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cd2f9a466065dd286ae3c9fbcf067da2, ASSIGN because future has completed 2024-11-21T17:43:13,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd2f9a466065dd286ae3c9fbcf067da2, server=f3865e8353b4,46273,1732210898278}] 2024-11-21T17:43:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T17:43:14,085 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:14,085 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => cd2f9a466065dd286ae3c9fbcf067da2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T17:43:14,085 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,086 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T17:43:14,086 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,086 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,088 INFO [StoreOpener-cd2f9a466065dd286ae3c9fbcf067da2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,089 INFO [StoreOpener-cd2f9a466065dd286ae3c9fbcf067da2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cd2f9a466065dd286ae3c9fbcf067da2 columnFamilyName testFamily 2024-11-21T17:43:14,089 DEBUG [StoreOpener-cd2f9a466065dd286ae3c9fbcf067da2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T17:43:14,090 INFO [StoreOpener-cd2f9a466065dd286ae3c9fbcf067da2-1 {}] regionserver.HStore(327): Store=cd2f9a466065dd286ae3c9fbcf067da2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T17:43:14,090 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,091 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,092 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,092 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,092 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,094 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,097 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T17:43:14,098 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened cd2f9a466065dd286ae3c9fbcf067da2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58787821, jitterRate=-0.12399320304393768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T17:43:14,098 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:14,099 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for cd2f9a466065dd286ae3c9fbcf067da2: Running coprocessor pre-open hook at 1732210994086Writing region info on filesystem at 1732210994086Initializing all the Stores at 1732210994087 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732210994087Cleaning up temporary data from old regions at 1732210994092 (+5 ms)Running coprocessor post-open hooks at 1732210994098 (+6 ms)Region opened successfully at 1732210994099 (+1 ms) 2024-11-21T17:43:14,100 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., pid=33, masterSystemTime=1732210994074 2024-11-21T17:43:14,103 DEBUG [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:14,103 INFO [RS_OPEN_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:14,104 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=cd2f9a466065dd286ae3c9fbcf067da2, regionState=OPEN, openSeqNum=2, regionLocation=f3865e8353b4,46273,1732210898278 2024-11-21T17:43:14,107 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure cd2f9a466065dd286ae3c9fbcf067da2, server=f3865e8353b4,46273,1732210898278 because future has completed 2024-11-21T17:43:14,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-21T17:43:14,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure cd2f9a466065dd286ae3c9fbcf067da2, server=f3865e8353b4,46273,1732210898278 in 188 msec 2024-11-21T17:43:14,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-21T17:43:14,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cd2f9a466065dd286ae3c9fbcf067da2, ASSIGN in 348 msec 2024-11-21T17:43:14,113 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T17:43:14,113 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732210994113"}]},"ts":"1732210994113"} 2024-11-21T17:43:14,115 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-21T17:43:14,116 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T17:43:14,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2000 sec 2024-11-21T17:43:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46457 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T17:43:15,966 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-21T17:43:15,966 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-21T17:43:15,966 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:43:15,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-21T17:43:15,977 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T17:43:15,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-21T17:43:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:43:15,991 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-21T17:43:15,991 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-21T17:43:15,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-21T17:43:15,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-21T17:43:16,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f3865e8353b4:46273 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-21T17:43:16,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-21T17:43:17,000 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 from cache 2024-11-21T17:43:17,079 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-21T17:43:17,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-21T17:43:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2] 2024-11-21T17:43:18,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46273 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-21T17:43:18,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-21T17:43:18,034 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-21T17:43:18,998 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=f3865e8353b4:46273 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=974ms 2024-11-21T17:43:18,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=974ms 2024-11-21T17:43:18,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., hostname=f3865e8353b4,46273,1732210898278, seqNum=2 from cache 2024-11-21T17:43:19,012 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 223), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=95 (was 104), ProcessCount=11 (was 11), AvailableMemoryMB=2209 (was 2216) 2024-11-21T17:43:19,013 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T17:43:19,013 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T17:43:19,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:43:19,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:43:19,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T17:43:19,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T17:43:19,019 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T17:43:19,020 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T17:43:19,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:43:19,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:43:19,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T17:43:19,020 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T17:43:19,020 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=291071084, stopped=false 2024-11-21T17:43:19,020 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f3865e8353b4,46457,1732210897685 2024-11-21T17:43:19,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T17:43:19,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T17:43:19,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:43:19,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:43:19,022 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T17:43:19,023 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T17:43:19,023 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T17:43:19,023 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T17:43:19,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:43:19,023 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T17:43:19,023 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f3865e8353b4,46273,1732210898278' ***** 2024-11-21T17:43:19,023 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T17:43:19,024 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T17:43:19,024 INFO [RS:0;f3865e8353b4:46273 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T17:43:19,024 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T17:43:19,024 INFO [RS:0;f3865e8353b4:46273 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T17:43:19,024 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for 4713e100e366f9b6cb42b459fb9993b3 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for c20801e25e613900913d41665a005d9c 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for 78559f7b90316348fa9afb96424506ab 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for 27ef19f9c166277483f3daf183d8a4da 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for 174ea425cc45b438f69b2ae95208b995 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for 6980b612a9341bf9afba0c7914664b14 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for 573f91880d4a7d271704d294e32c1169 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for d43585d0894151ec1d56b74d358e2b3b 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:19,025 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4713e100e366f9b6cb42b459fb9993b3, disabling compactions & flushes 2024-11-21T17:43:19,025 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:19,026 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:43:19,026 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(959): stopping server f3865e8353b4,46273,1732210898278 2024-11-21T17:43:19,026 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:43:19,026 INFO [RS:0;f3865e8353b4:46273 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T17:43:19,026 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. after waiting 0 ms 2024-11-21T17:43:19,026 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:43:19,026 INFO [RS:0;f3865e8353b4:46273 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f3865e8353b4:46273. 2024-11-21T17:43:19,026 DEBUG [RS:0;f3865e8353b4:46273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T17:43:19,026 DEBUG [RS:0;f3865e8353b4:46273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:43:19,026 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T17:43:19,026 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T17:43:19,026 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T17:43:19,026 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T17:43:19,027 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-21T17:43:19,027 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T17:43:19,027 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T17:43:19,027 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T17:43:19,027 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T17:43:19,027 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T17:43:19,027 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 174ea425cc45b438f69b2ae95208b995, 27ef19f9c166277483f3daf183d8a4da, 4713e100e366f9b6cb42b459fb9993b3, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, 78559f7b90316348fa9afb96424506ab, c20801e25e613900913d41665a005d9c, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:19,029 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4713e100e366f9b6cb42b459fb9993b3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:19,029 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-21T17:43:19,078 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3/.tmp/testFamily/d634074cb76b4983a7fd99137c983699 is 61, key is row-0/testFamily:testQualifier/1732210938063/Put/seqid=0 2024-11-21T17:43:19,087 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/info/3d63d4236e24465e9694041b1d54f240 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169./info:regioninfo/1732210933069/Put/seqid=0 2024-11-21T17:43:19,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741855_1031 (size=5049) 2024-11-21T17:43:19,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741856_1032 (size=20564) 2024-11-21T17:43:19,227 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 174ea425cc45b438f69b2ae95208b995, 27ef19f9c166277483f3daf183d8a4da, 4713e100e366f9b6cb42b459fb9993b3, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, 78559f7b90316348fa9afb96424506ab, c20801e25e613900913d41665a005d9c, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:19,428 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 174ea425cc45b438f69b2ae95208b995, 27ef19f9c166277483f3daf183d8a4da, 4713e100e366f9b6cb42b459fb9993b3, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, 78559f7b90316348fa9afb96424506ab, c20801e25e613900913d41665a005d9c, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:19,490 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3/.tmp/testFamily/d634074cb76b4983a7fd99137c983699 2024-11-21T17:43:19,495 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/info/3d63d4236e24465e9694041b1d54f240 2024-11-21T17:43:19,524 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-21T17:43:19,539 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3/.tmp/testFamily/d634074cb76b4983a7fd99137c983699 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3/testFamily/d634074cb76b4983a7fd99137c983699 2024-11-21T17:43:19,548 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3/testFamily/d634074cb76b4983a7fd99137c983699, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:19,556 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4713e100e366f9b6cb42b459fb9993b3 in 527ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:19,561 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/ns/b05d0246e9494cabb7b32ac025b2723c is 43, key is default/ns:d/1732210931599/Put/seqid=0 2024-11-21T17:43:19,563 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScanner/4713e100e366f9b6cb42b459fb9993b3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:19,566 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:43:19,566 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4713e100e366f9b6cb42b459fb9993b3: Waiting for close lock at 1732210999025Running coprocessor pre-close hooks at 1732210999025Disabling compacts and flushes for region at 1732210999025Disabling writes for close at 1732210999026 (+1 ms)Obtaining lock to block concurrent updates at 1732210999029 (+3 ms)Preparing flush snapshotting stores in 4713e100e366f9b6cb42b459fb9993b3 at 1732210999029Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732210999035 (+6 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. at 1732210999037 (+2 ms)Flushing 4713e100e366f9b6cb42b459fb9993b3/testFamily: creating writer at 1732210999038 (+1 ms)Flushing 4713e100e366f9b6cb42b459fb9993b3/testFamily: appending metadata at 1732210999073 (+35 ms)Flushing 4713e100e366f9b6cb42b459fb9993b3/testFamily: closing flushed file at 1732210999075 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78e90930: reopening flushed file at 1732210999537 (+462 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4713e100e366f9b6cb42b459fb9993b3 in 527ms, sequenceid=8, compaction requested=false at 1732210999556 (+19 ms)Writing region close event to WAL at 1732210999558 (+2 ms)Running coprocessor post-close hooks at 1732210999564 (+6 ms)Closed at 1732210999565 (+1 ms) 2024-11-21T17:43:19,566 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732210935019.4713e100e366f9b6cb42b459fb9993b3. 2024-11-21T17:43:19,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741857_1033 (size=5153) 2024-11-21T17:43:19,567 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c20801e25e613900913d41665a005d9c, disabling compactions & flushes 2024-11-21T17:43:19,567 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:43:19,567 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:43:19,567 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. after waiting 0 ms 2024-11-21T17:43:19,567 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:43:19,567 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c20801e25e613900913d41665a005d9c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:19,570 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/ns/b05d0246e9494cabb7b32ac025b2723c 2024-11-21T17:43:19,587 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:43:19,587 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c/.tmp/testFamily/25d3c141fa94488f925a5f865b9a7748 is 61, key is row-0/testFamily:testQualifier/1732210965533/Put/seqid=0 2024-11-21T17:43:19,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741858_1034 (size=5049) 2024-11-21T17:43:19,594 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c/.tmp/testFamily/25d3c141fa94488f925a5f865b9a7748 2024-11-21T17:43:19,599 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/table/079b1d83bfdf4c2f895513ff39a19c52 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732210933086/Put/seqid=0 2024-11-21T17:43:19,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741859_1035 (size=6699) 2024-11-21T17:43:19,606 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/table/079b1d83bfdf4c2f895513ff39a19c52 2024-11-21T17:43:19,606 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c/.tmp/testFamily/25d3c141fa94488f925a5f865b9a7748 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c/testFamily/25d3c141fa94488f925a5f865b9a7748 2024-11-21T17:43:19,615 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c/testFamily/25d3c141fa94488f925a5f865b9a7748, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:19,616 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c20801e25e613900913d41665a005d9c in 49ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:19,616 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/info/3d63d4236e24465e9694041b1d54f240 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/info/3d63d4236e24465e9694041b1d54f240 2024-11-21T17:43:19,623 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNextAsync/c20801e25e613900913d41665a005d9c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:19,624 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:43:19,624 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c20801e25e613900913d41665a005d9c: Waiting for close lock at 1732210999567Running coprocessor pre-close hooks at 1732210999567Disabling compacts and flushes for region at 1732210999567Disabling writes for close at 1732210999567Obtaining lock to block concurrent updates at 1732210999567Preparing flush snapshotting stores in c20801e25e613900913d41665a005d9c at 1732210999567Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732210999567Flushing stores of testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. at 1732210999568 (+1 ms)Flushing c20801e25e613900913d41665a005d9c/testFamily: creating writer at 1732210999568Flushing c20801e25e613900913d41665a005d9c/testFamily: appending metadata at 1732210999587 (+19 ms)Flushing c20801e25e613900913d41665a005d9c/testFamily: closing flushed file at 1732210999587Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101f5a91: reopening flushed file at 1732210999604 (+17 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c20801e25e613900913d41665a005d9c in 49ms, sequenceid=8, compaction requested=false at 1732210999616 (+12 ms)Writing region close event to WAL at 1732210999617 (+1 ms)Running coprocessor post-close hooks at 1732210999624 (+7 ms)Closed at 1732210999624 2024-11-21T17:43:19,624 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732210962462.c20801e25e613900913d41665a005d9c. 2024-11-21T17:43:19,624 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 78559f7b90316348fa9afb96424506ab, disabling compactions & flushes 2024-11-21T17:43:19,625 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:43:19,625 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:43:19,625 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. after waiting 0 ms 2024-11-21T17:43:19,625 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:43:19,625 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 78559f7b90316348fa9afb96424506ab 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:19,627 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/info/3d63d4236e24465e9694041b1d54f240, entries=100, sequenceid=56, filesize=20.1 K 2024-11-21T17:43:19,628 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 174ea425cc45b438f69b2ae95208b995, 27ef19f9c166277483f3daf183d8a4da, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, 78559f7b90316348fa9afb96424506ab, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:19,629 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/ns/b05d0246e9494cabb7b32ac025b2723c as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/ns/b05d0246e9494cabb7b32ac025b2723c 2024-11-21T17:43:19,639 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/ns/b05d0246e9494cabb7b32ac025b2723c, entries=2, sequenceid=56, filesize=5.0 K 2024-11-21T17:43:19,640 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/.tmp/table/079b1d83bfdf4c2f895513ff39a19c52 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/table/079b1d83bfdf4c2f895513ff39a19c52 2024-11-21T17:43:19,644 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab/.tmp/testFamily/3e21ccc80748449db18b39003d2ff005 is 61, key is row-0/testFamily:testQualifier/1732210971651/Put/seqid=0 2024-11-21T17:43:19,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741860_1036 (size=5049) 2024-11-21T17:43:19,649 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab/.tmp/testFamily/3e21ccc80748449db18b39003d2ff005 2024-11-21T17:43:19,650 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/table/079b1d83bfdf4c2f895513ff39a19c52, entries=20, sequenceid=56, filesize=6.5 K 2024-11-21T17:43:19,651 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 624ms, sequenceid=56, compaction requested=false 2024-11-21T17:43:19,657 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-21T17:43:19,658 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T17:43:19,658 INFO [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T17:43:19,659 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732210999027Running coprocessor pre-close hooks at 1732210999027Disabling compacts and flushes for region at 1732210999027Disabling writes for close at 1732210999027Obtaining lock to block concurrent updates at 1732210999029 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732210999029Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732210999035 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732210999037 (+2 ms)Flushing 1588230740/info: creating writer at 1732210999038 (+1 ms)Flushing 1588230740/info: appending metadata at 1732210999080 (+42 ms)Flushing 1588230740/info: closing flushed file at 1732210999080Flushing 1588230740/ns: creating writer at 1732210999537 (+457 ms)Flushing 1588230740/ns: appending metadata at 1732210999559 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732210999559Flushing 1588230740/table: creating writer at 1732210999579 (+20 ms)Flushing 1588230740/table: appending metadata at 1732210999598 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732210999598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@510d1aa5: reopening flushed file at 1732210999615 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@796ca182: reopening flushed file at 1732210999627 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ea962dd: reopening flushed file at 1732210999639 (+12 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 624ms, sequenceid=56, compaction requested=false at 1732210999651 (+12 ms)Writing region close event to WAL at 1732210999653 (+2 ms)Running coprocessor post-close hooks at 1732210999658 (+5 ms)Closed at 1732210999658 2024-11-21T17:43:19,659 DEBUG [RS_CLOSE_META-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T17:43:19,660 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab/.tmp/testFamily/3e21ccc80748449db18b39003d2ff005 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab/testFamily/3e21ccc80748449db18b39003d2ff005 2024-11-21T17:43:19,669 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab/testFamily/3e21ccc80748449db18b39003d2ff005, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:19,670 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 78559f7b90316348fa9afb96424506ab in 45ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:19,676 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnNext/78559f7b90316348fa9afb96424506ab/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:19,676 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:43:19,676 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 78559f7b90316348fa9afb96424506ab: Waiting for close lock at 1732210999624Running coprocessor pre-close hooks at 1732210999624Disabling compacts and flushes for region at 1732210999624Disabling writes for close at 1732210999625 (+1 ms)Obtaining lock to block concurrent updates at 1732210999625Preparing flush snapshotting stores in 78559f7b90316348fa9afb96424506ab at 1732210999625Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732210999625Flushing stores of testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. at 1732210999626 (+1 ms)Flushing 78559f7b90316348fa9afb96424506ab/testFamily: creating writer at 1732210999626Flushing 78559f7b90316348fa9afb96424506ab/testFamily: appending metadata at 1732210999644 (+18 ms)Flushing 78559f7b90316348fa9afb96424506ab/testFamily: closing flushed file at 1732210999644Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10d04e6: reopening flushed file at 1732210999659 (+15 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 78559f7b90316348fa9afb96424506ab in 45ms, sequenceid=8, compaction requested=false at 1732210999670 (+11 ms)Writing region close event to WAL at 1732210999671 (+1 ms)Running coprocessor post-close hooks at 1732210999676 (+5 ms)Closed at 1732210999676 2024-11-21T17:43:19,677 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732210968587.78559f7b90316348fa9afb96424506ab. 2024-11-21T17:43:19,677 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 27ef19f9c166277483f3daf183d8a4da, disabling compactions & flushes 2024-11-21T17:43:19,677 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:19,677 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:19,677 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. after waiting 0 ms 2024-11-21T17:43:19,677 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:19,677 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 27ef19f9c166277483f3daf183d8a4da 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:19,700 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da/.tmp/testFamily/065a055f4ef347238c035c5ddd822d56 is 61, key is row-0/testFamily:testQualifier/1732210986862/Put/seqid=0 2024-11-21T17:43:19,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741861_1037 (size=5049) 2024-11-21T17:43:19,786 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=f3865e8353b4:46457 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 10 more 2024-11-21T17:43:19,828 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 174ea425cc45b438f69b2ae95208b995, 27ef19f9c166277483f3daf183d8a4da, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:19,967 INFO [regionserver/f3865e8353b4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T17:43:19,967 INFO [regionserver/f3865e8353b4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T17:43:20,029 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-11-21T17:43:20,029 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1325): Online Regions={27ef19f9c166277483f3daf183d8a4da=hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da., 174ea425cc45b438f69b2ae95208b995=hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995., 6980b612a9341bf9afba0c7914664b14=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., 573f91880d4a7d271704d294e32c1169=testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169., d43585d0894151ec1d56b74d358e2b3b=testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b., d49dabad34469f3e689a915ad669f6d4=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., cd2f9a466065dd286ae3c9fbcf067da2=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.} 2024-11-21T17:43:20,029 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 174ea425cc45b438f69b2ae95208b995, 27ef19f9c166277483f3daf183d8a4da, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:20,107 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da/.tmp/testFamily/065a055f4ef347238c035c5ddd822d56 2024-11-21T17:43:20,121 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da/.tmp/testFamily/065a055f4ef347238c035c5ddd822d56 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da/testFamily/065a055f4ef347238c035c5ddd822d56 2024-11-21T17:43:20,128 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da/testFamily/065a055f4ef347238c035c5ddd822d56, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:20,129 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 27ef19f9c166277483f3daf183d8a4da in 452ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:20,133 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNext/27ef19f9c166277483f3daf183d8a4da/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:20,134 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:20,134 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 27ef19f9c166277483f3daf183d8a4da: Waiting for close lock at 1732210999677Running coprocessor pre-close hooks at 1732210999677Disabling compacts and flushes for region at 1732210999677Disabling writes for close at 1732210999677Obtaining lock to block concurrent updates at 1732210999677Preparing flush snapshotting stores in 27ef19f9c166277483f3daf183d8a4da at 1732210999677Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732210999677Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. at 1732210999678 (+1 ms)Flushing 27ef19f9c166277483f3daf183d8a4da/testFamily: creating writer at 1732210999678Flushing 27ef19f9c166277483f3daf183d8a4da/testFamily: appending metadata at 1732210999699 (+21 ms)Flushing 27ef19f9c166277483f3daf183d8a4da/testFamily: closing flushed file at 1732210999699Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6efdfa4d: reopening flushed file at 1732211000119 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 27ef19f9c166277483f3daf183d8a4da in 452ms, sequenceid=8, compaction requested=false at 1732211000129 (+10 ms)Writing region close event to WAL at 1732211000129Running coprocessor post-close hooks at 1732211000134 (+5 ms)Closed at 1732211000134 2024-11-21T17:43:20,134 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732210983806.27ef19f9c166277483f3daf183d8a4da. 2024-11-21T17:43:20,135 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 174ea425cc45b438f69b2ae95208b995, disabling compactions & flushes 2024-11-21T17:43:20,135 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:43:20,135 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:43:20,135 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. after waiting 0 ms 2024-11-21T17:43:20,135 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:43:20,135 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 174ea425cc45b438f69b2ae95208b995 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:20,152 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995/.tmp/testFamily/f46e2a9ffde741f5acbb580aeff3dfe7 is 61, key is row-0/testFamily:testQualifier/1732210947191/Put/seqid=0 2024-11-21T17:43:20,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741862_1038 (size=5049) 2024-11-21T17:43:20,230 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 174ea425cc45b438f69b2ae95208b995, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:20,430 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 174ea425cc45b438f69b2ae95208b995, 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:20,560 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995/.tmp/testFamily/f46e2a9ffde741f5acbb580aeff3dfe7 2024-11-21T17:43:20,572 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995/.tmp/testFamily/f46e2a9ffde741f5acbb580aeff3dfe7 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995/testFamily/f46e2a9ffde741f5acbb580aeff3dfe7 2024-11-21T17:43:20,579 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995/testFamily/f46e2a9ffde741f5acbb580aeff3dfe7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:20,580 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 174ea425cc45b438f69b2ae95208b995 in 445ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:20,585 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnNextAsync/174ea425cc45b438f69b2ae95208b995/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:20,586 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:43:20,586 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 174ea425cc45b438f69b2ae95208b995: Waiting for close lock at 1732211000135Running coprocessor pre-close hooks at 1732211000135Disabling compacts and flushes for region at 1732211000135Disabling writes for close at 1732211000135Obtaining lock to block concurrent updates at 1732211000135Preparing flush snapshotting stores in 174ea425cc45b438f69b2ae95208b995 at 1732211000135Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732211000135Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. at 1732211000136 (+1 ms)Flushing 174ea425cc45b438f69b2ae95208b995/testFamily: creating writer at 1732211000136Flushing 174ea425cc45b438f69b2ae95208b995/testFamily: appending metadata at 1732211000152 (+16 ms)Flushing 174ea425cc45b438f69b2ae95208b995/testFamily: closing flushed file at 1732211000152Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@429d6e2b: reopening flushed file at 1732211000571 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 174ea425cc45b438f69b2ae95208b995 in 445ms, sequenceid=8, compaction requested=false at 1732211000580 (+9 ms)Writing region close event to WAL at 1732211000581 (+1 ms)Running coprocessor post-close hooks at 1732211000586 (+5 ms)Closed at 1732211000586 2024-11-21T17:43:20,586 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732210944121.174ea425cc45b438f69b2ae95208b995. 2024-11-21T17:43:20,586 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6980b612a9341bf9afba0c7914664b14, disabling compactions & flushes 2024-11-21T17:43:20,586 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:43:20,586 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:43:20,586 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. after waiting 0 ms 2024-11-21T17:43:20,586 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:43:20,586 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6980b612a9341bf9afba0c7914664b14 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:20,602 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14/.tmp/testFamily/6589e17f4e034762b9d160e096321468 is 61, key is row-0/testFamily:testQualifier/1732210977759/Put/seqid=0 2024-11-21T17:43:20,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741863_1039 (size=5049) 2024-11-21T17:43:20,630 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:20,831 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:21,010 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14/.tmp/testFamily/6589e17f4e034762b9d160e096321468 2024-11-21T17:43:21,025 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14/.tmp/testFamily/6589e17f4e034762b9d160e096321468 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14/testFamily/6589e17f4e034762b9d160e096321468 2024-11-21T17:43:21,031 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-21T17:43:21,031 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1325): Online Regions={6980b612a9341bf9afba0c7914664b14=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., 573f91880d4a7d271704d294e32c1169=testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169., d43585d0894151ec1d56b74d358e2b3b=testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b., d49dabad34469f3e689a915ad669f6d4=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., cd2f9a466065dd286ae3c9fbcf067da2=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.} 2024-11-21T17:43:21,031 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 573f91880d4a7d271704d294e32c1169, 6980b612a9341bf9afba0c7914664b14, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:21,034 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14/testFamily/6589e17f4e034762b9d160e096321468, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:21,035 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6980b612a9341bf9afba0c7914664b14 in 449ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:21,039 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/6980b612a9341bf9afba0c7914664b14/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:21,040 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:43:21,040 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6980b612a9341bf9afba0c7914664b14: Waiting for close lock at 1732211000586Running coprocessor pre-close hooks at 1732211000586Disabling compacts and flushes for region at 1732211000586Disabling writes for close at 1732211000586Obtaining lock to block concurrent updates at 1732211000587 (+1 ms)Preparing flush snapshotting stores in 6980b612a9341bf9afba0c7914664b14 at 1732211000587Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732211000587Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. at 1732211000587Flushing 6980b612a9341bf9afba0c7914664b14/testFamily: creating writer at 1732211000588 (+1 ms)Flushing 6980b612a9341bf9afba0c7914664b14/testFamily: appending metadata at 1732211000601 (+13 ms)Flushing 6980b612a9341bf9afba0c7914664b14/testFamily: closing flushed file at 1732211000602 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@581636fc: reopening flushed file at 1732211001024 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6980b612a9341bf9afba0c7914664b14 in 449ms, sequenceid=8, compaction requested=false at 1732211001035 (+11 ms)Writing region close event to WAL at 1732211001035Running coprocessor post-close hooks at 1732211001040 (+5 ms)Closed at 1732211001040 2024-11-21T17:43:21,040 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732210974708.6980b612a9341bf9afba0c7914664b14. 2024-11-21T17:43:21,040 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 573f91880d4a7d271704d294e32c1169, disabling compactions & flushes 2024-11-21T17:43:21,040 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:43:21,040 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:43:21,040 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. after waiting 0 ms 2024-11-21T17:43:21,040 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:43:21,040 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 573f91880d4a7d271704d294e32c1169 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:21,056 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169/.tmp/testFamily/1f889e8eb7074a51afb881461b3ca7c8 is 61, key is row-0/testFamily:testQualifier/1732210934923/Put/seqid=0 2024-11-21T17:43:21,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741864_1040 (size=5049) 2024-11-21T17:43:21,232 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 573f91880d4a7d271704d294e32c1169, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:21,432 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on 573f91880d4a7d271704d294e32c1169, cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:21,462 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169/.tmp/testFamily/1f889e8eb7074a51afb881461b3ca7c8 2024-11-21T17:43:21,478 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169/.tmp/testFamily/1f889e8eb7074a51afb881461b3ca7c8 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169/testFamily/1f889e8eb7074a51afb881461b3ca7c8 2024-11-21T17:43:21,487 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169/testFamily/1f889e8eb7074a51afb881461b3ca7c8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:21,488 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 573f91880d4a7d271704d294e32c1169 in 448ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:21,493 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextExceptionAsync/573f91880d4a7d271704d294e32c1169/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:21,494 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:43:21,494 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 573f91880d4a7d271704d294e32c1169: Waiting for close lock at 1732211001040Running coprocessor pre-close hooks at 1732211001040Disabling compacts and flushes for region at 1732211001040Disabling writes for close at 1732211001040Obtaining lock to block concurrent updates at 1732211001041 (+1 ms)Preparing flush snapshotting stores in 573f91880d4a7d271704d294e32c1169 at 1732211001041Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732211001041Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. at 1732211001041Flushing 573f91880d4a7d271704d294e32c1169/testFamily: creating writer at 1732211001042 (+1 ms)Flushing 573f91880d4a7d271704d294e32c1169/testFamily: appending metadata at 1732211001055 (+13 ms)Flushing 573f91880d4a7d271704d294e32c1169/testFamily: closing flushed file at 1732211001055Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77a09ec3: reopening flushed file at 1732211001476 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 573f91880d4a7d271704d294e32c1169 in 448ms, sequenceid=8, compaction requested=false at 1732211001488 (+12 ms)Writing region close event to WAL at 1732211001489 (+1 ms)Running coprocessor post-close hooks at 1732211001494 (+5 ms)Closed at 1732211001494 2024-11-21T17:43:21,494 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732210931835.573f91880d4a7d271704d294e32c1169. 2024-11-21T17:43:21,494 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d43585d0894151ec1d56b74d358e2b3b, disabling compactions & flushes 2024-11-21T17:43:21,494 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:43:21,494 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:43:21,494 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. after waiting 0 ms 2024-11-21T17:43:21,494 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:43:21,494 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d43585d0894151ec1d56b74d358e2b3b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:21,510 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b/.tmp/testFamily/be58c9ab862b40ca8b846605f624e298 is 61, key is row-0/testFamily:testQualifier/1732210956310/Put/seqid=0 2024-11-21T17:43:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741865_1041 (size=5049) 2024-11-21T17:43:21,632 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:21,833 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on cd2f9a466065dd286ae3c9fbcf067da2, d43585d0894151ec1d56b74d358e2b3b, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:21,916 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b/.tmp/testFamily/be58c9ab862b40ca8b846605f624e298 2024-11-21T17:43:21,930 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b/.tmp/testFamily/be58c9ab862b40ca8b846605f624e298 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b/testFamily/be58c9ab862b40ca8b846605f624e298 2024-11-21T17:43:21,938 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b/testFamily/be58c9ab862b40ca8b846605f624e298, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:21,939 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d43585d0894151ec1d56b74d358e2b3b in 445ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:21,944 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testRetryOutOfOrderScannerNextException/d43585d0894151ec1d56b74d358e2b3b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:21,945 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:43:21,945 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d43585d0894151ec1d56b74d358e2b3b: Waiting for close lock at 1732211001494Running coprocessor pre-close hooks at 1732211001494Disabling compacts and flushes for region at 1732211001494Disabling writes for close at 1732211001494Obtaining lock to block concurrent updates at 1732211001494Preparing flush snapshotting stores in d43585d0894151ec1d56b74d358e2b3b at 1732211001494Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732211001495 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. at 1732211001495Flushing d43585d0894151ec1d56b74d358e2b3b/testFamily: creating writer at 1732211001496 (+1 ms)Flushing d43585d0894151ec1d56b74d358e2b3b/testFamily: appending metadata at 1732211001509 (+13 ms)Flushing d43585d0894151ec1d56b74d358e2b3b/testFamily: closing flushed file at 1732211001510 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a5082c8: reopening flushed file at 1732211001929 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d43585d0894151ec1d56b74d358e2b3b in 445ms, sequenceid=8, compaction requested=false at 1732211001939 (+10 ms)Writing region close event to WAL at 1732211001940 (+1 ms)Running coprocessor post-close hooks at 1732211001945 (+5 ms)Closed at 1732211001945 2024-11-21T17:43:21,945 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732210953247.d43585d0894151ec1d56b74d358e2b3b. 2024-11-21T17:43:21,945 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d49dabad34469f3e689a915ad669f6d4, disabling compactions & flushes 2024-11-21T17:43:21,945 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:43:21,945 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:43:21,945 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. after waiting 0 ms 2024-11-21T17:43:21,945 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:43:21,946 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d49dabad34469f3e689a915ad669f6d4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:21,962 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4/.tmp/testFamily/46b4a01f99ef4004af7bba41cc58b68b is 61, key is row-0/testFamily:testQualifier/1732210959420/Put/seqid=0 2024-11-21T17:43:21,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741866_1042 (size=5049) 2024-11-21T17:43:22,033 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T17:43:22,033 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1325): Online Regions={d49dabad34469f3e689a915ad669f6d4=testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., cd2f9a466065dd286ae3c9fbcf067da2=testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2.} 2024-11-21T17:43:22,033 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on cd2f9a466065dd286ae3c9fbcf067da2, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:22,234 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on cd2f9a466065dd286ae3c9fbcf067da2, d49dabad34469f3e689a915ad669f6d4 2024-11-21T17:43:22,368 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4/.tmp/testFamily/46b4a01f99ef4004af7bba41cc58b68b 2024-11-21T17:43:22,381 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4/.tmp/testFamily/46b4a01f99ef4004af7bba41cc58b68b as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4/testFamily/46b4a01f99ef4004af7bba41cc58b68b 2024-11-21T17:43:22,392 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4/testFamily/46b4a01f99ef4004af7bba41cc58b68b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:22,393 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d49dabad34469f3e689a915ad669f6d4 in 448ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:22,398 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScanner/d49dabad34469f3e689a915ad669f6d4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:22,399 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:43:22,399 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d49dabad34469f3e689a915ad669f6d4: Waiting for close lock at 1732211001945Running coprocessor pre-close hooks at 1732211001945Disabling compacts and flushes for region at 1732211001945Disabling writes for close at 1732211001945Obtaining lock to block concurrent updates at 1732211001946 (+1 ms)Preparing flush snapshotting stores in d49dabad34469f3e689a915ad669f6d4 at 1732211001946Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732211001946Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. at 1732211001946Flushing d49dabad34469f3e689a915ad669f6d4/testFamily: creating writer at 1732211001946Flushing d49dabad34469f3e689a915ad669f6d4/testFamily: appending metadata at 1732211001961 (+15 ms)Flushing d49dabad34469f3e689a915ad669f6d4/testFamily: closing flushed file at 1732211001961Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@425feb1b: reopening flushed file at 1732211002380 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d49dabad34469f3e689a915ad669f6d4 in 448ms, sequenceid=8, compaction requested=false at 1732211002393 (+13 ms)Writing region close event to WAL at 1732211002394 (+1 ms)Running coprocessor post-close hooks at 1732211002398 (+4 ms)Closed at 1732211002398 2024-11-21T17:43:22,399 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732210956361.d49dabad34469f3e689a915ad669f6d4. 2024-11-21T17:43:22,399 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cd2f9a466065dd286ae3c9fbcf067da2, disabling compactions & flushes 2024-11-21T17:43:22,399 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:22,399 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:22,399 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. after waiting 0 ms 2024-11-21T17:43:22,399 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:22,399 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cd2f9a466065dd286ae3c9fbcf067da2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-21T17:43:22,416 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2/.tmp/testFamily/b3440a1dd01245d3b1783d7eca8a77f8 is 61, key is row-0/testFamily:testQualifier/1732210995982/Put/seqid=0 2024-11-21T17:43:22,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741867_1043 (size=5049) 2024-11-21T17:43:22,434 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:22,634 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:22,824 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2/.tmp/testFamily/b3440a1dd01245d3b1783d7eca8a77f8 2024-11-21T17:43:22,834 DEBUG [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1351): Waiting on cd2f9a466065dd286ae3c9fbcf067da2 2024-11-21T17:43:22,837 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2/.tmp/testFamily/b3440a1dd01245d3b1783d7eca8a77f8 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2/testFamily/b3440a1dd01245d3b1783d7eca8a77f8 2024-11-21T17:43:22,844 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2/testFamily/b3440a1dd01245d3b1783d7eca8a77f8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-21T17:43:22,845 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cd2f9a466065dd286ae3c9fbcf067da2 in 446ms, sequenceid=8, compaction requested=false 2024-11-21T17:43:22,849 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/data/default/testNormalScanTimeoutOnOpenScannerAsync/cd2f9a466065dd286ae3c9fbcf067da2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-21T17:43:22,850 INFO [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:22,850 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cd2f9a466065dd286ae3c9fbcf067da2: Waiting for close lock at 1732211002399Running coprocessor pre-close hooks at 1732211002399Disabling compacts and flushes for region at 1732211002399Disabling writes for close at 1732211002399Obtaining lock to block concurrent updates at 1732211002399Preparing flush snapshotting stores in cd2f9a466065dd286ae3c9fbcf067da2 at 1732211002399Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732211002399Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. at 1732211002400 (+1 ms)Flushing cd2f9a466065dd286ae3c9fbcf067da2/testFamily: creating writer at 1732211002400Flushing cd2f9a466065dd286ae3c9fbcf067da2/testFamily: appending metadata at 1732211002416 (+16 ms)Flushing cd2f9a466065dd286ae3c9fbcf067da2/testFamily: closing flushed file at 1732211002416Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ff7d0a6: reopening flushed file at 1732211002836 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cd2f9a466065dd286ae3c9fbcf067da2 in 446ms, sequenceid=8, compaction requested=false at 1732211002845 (+9 ms)Writing region close event to WAL at 1732211002845Running coprocessor post-close hooks at 1732211002850 (+5 ms)Closed at 1732211002850 2024-11-21T17:43:22,850 DEBUG [RS_CLOSE_REGION-regionserver/f3865e8353b4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732210992916.cd2f9a466065dd286ae3c9fbcf067da2. 2024-11-21T17:43:23,035 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(976): stopping server f3865e8353b4,46273,1732210898278; all regions closed. 2024-11-21T17:43:23,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741834_1010 (size=19479) 2024-11-21T17:43:23,050 DEBUG [RS:0;f3865e8353b4:46273 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/oldWALs 2024-11-21T17:43:23,051 INFO [RS:0;f3865e8353b4:46273 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f3865e8353b4%2C46273%2C1732210898278.meta:.meta(num 1732210931454) 2024-11-21T17:43:23,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741833_1009 (size=20031) 2024-11-21T17:43:23,056 DEBUG [RS:0;f3865e8353b4:46273 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/oldWALs 2024-11-21T17:43:23,056 INFO [RS:0;f3865e8353b4:46273 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f3865e8353b4%2C46273%2C1732210898278:(num 1732210931002) 2024-11-21T17:43:23,056 DEBUG [RS:0;f3865e8353b4:46273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T17:43:23,056 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T17:43:23,056 INFO [RS:0;f3865e8353b4:46273 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T17:43:23,057 INFO [RS:0;f3865e8353b4:46273 {}] hbase.ChoreService(370): Chore service for: regionserver/f3865e8353b4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T17:43:23,057 INFO [RS:0;f3865e8353b4:46273 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T17:43:23,057 INFO [regionserver/f3865e8353b4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T17:43:23,058 INFO [RS:0;f3865e8353b4:46273 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46273 2024-11-21T17:43:23,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f3865e8353b4,46273,1732210898278 2024-11-21T17:43:23,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T17:43:23,061 INFO [RS:0;f3865e8353b4:46273 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T17:43:23,063 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f3865e8353b4,46273,1732210898278] 2024-11-21T17:43:23,064 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f3865e8353b4,46273,1732210898278 already deleted, retry=false 2024-11-21T17:43:23,064 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f3865e8353b4,46273,1732210898278 expired; onlineServers=0 2024-11-21T17:43:23,064 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f3865e8353b4,46457,1732210897685' ***** 2024-11-21T17:43:23,065 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T17:43:23,065 INFO [M:0;f3865e8353b4:46457 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T17:43:23,065 INFO [M:0;f3865e8353b4:46457 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T17:43:23,065 DEBUG [M:0;f3865e8353b4:46457 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T17:43:23,065 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T17:43:23,065 DEBUG [M:0;f3865e8353b4:46457 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T17:43:23,065 DEBUG [master/f3865e8353b4:0:becomeActiveMaster-HFileCleaner.small.0-1732210900800 {}] cleaner.HFileCleaner(306): Exit Thread[master/f3865e8353b4:0:becomeActiveMaster-HFileCleaner.small.0-1732210900800,5,FailOnTimeoutGroup] 2024-11-21T17:43:23,065 DEBUG [master/f3865e8353b4:0:becomeActiveMaster-HFileCleaner.large.0-1732210900796 {}] cleaner.HFileCleaner(306): Exit Thread[master/f3865e8353b4:0:becomeActiveMaster-HFileCleaner.large.0-1732210900796,5,FailOnTimeoutGroup] 2024-11-21T17:43:23,065 INFO [M:0;f3865e8353b4:46457 {}] hbase.ChoreService(370): Chore service for: master/f3865e8353b4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T17:43:23,065 INFO [M:0;f3865e8353b4:46457 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T17:43:23,066 DEBUG [M:0;f3865e8353b4:46457 {}] master.HMaster(1795): Stopping service threads 2024-11-21T17:43:23,066 INFO [M:0;f3865e8353b4:46457 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T17:43:23,066 INFO [M:0;f3865e8353b4:46457 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T17:43:23,066 INFO [M:0;f3865e8353b4:46457 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T17:43:23,066 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T17:43:23,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T17:43:23,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T17:43:23,067 DEBUG [M:0;f3865e8353b4:46457 {}] zookeeper.ZKUtil(347): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T17:43:23,067 WARN [M:0;f3865e8353b4:46457 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T17:43:23,068 INFO [M:0;f3865e8353b4:46457 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/.lastflushedseqids 2024-11-21T17:43:23,076 INFO [regionserver/f3865e8353b4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T17:43:23,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741868_1044 (size=649) 2024-11-21T17:43:23,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T17:43:23,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46273-0x10063600c730001, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T17:43:23,164 INFO [RS:0;f3865e8353b4:46273 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T17:43:23,164 INFO [RS:0;f3865e8353b4:46273 {}] regionserver.HRegionServer(1031): Exiting; stopping=f3865e8353b4,46273,1732210898278; zookeeper connection closed. 2024-11-21T17:43:23,165 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c723667 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c723667 2024-11-21T17:43:23,166 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T17:43:23,480 INFO [M:0;f3865e8353b4:46457 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T17:43:23,481 INFO [M:0;f3865e8353b4:46457 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T17:43:23,481 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T17:43:23,482 INFO [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:43:23,482 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:43:23,482 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T17:43:23,482 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:43:23,483 INFO [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.98 KB heapSize=190.87 KB 2024-11-21T17:43:23,503 DEBUG [M:0;f3865e8353b4:46457 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74e414eddab148b589b79f6e8c26b848 is 82, key is hbase:meta,,1/info:regioninfo/1732210931536/Put/seqid=0 2024-11-21T17:43:23,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741869_1045 (size=5672) 2024-11-21T17:43:23,910 INFO [M:0;f3865e8353b4:46457 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74e414eddab148b589b79f6e8c26b848 2024-11-21T17:43:23,945 DEBUG [M:0;f3865e8353b4:46457 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/56e69f58722648d8819c0d93fd099a43 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732210933093/Put/seqid=0 2024-11-21T17:43:23,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741870_1046 (size=14069) 2024-11-21T17:43:24,353 INFO [M:0;f3865e8353b4:46457 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.42 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/56e69f58722648d8819c0d93fd099a43 2024-11-21T17:43:24,367 INFO [M:0;f3865e8353b4:46457 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56e69f58722648d8819c0d93fd099a43 2024-11-21T17:43:24,384 DEBUG [M:0;f3865e8353b4:46457 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c4f03ee053340a59b306d078fb5bae9 is 69, key is f3865e8353b4,46273,1732210898278/rs:state/1732210900851/Put/seqid=0 2024-11-21T17:43:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741871_1047 (size=5156) 2024-11-21T17:43:24,790 INFO [M:0;f3865e8353b4:46457 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c4f03ee053340a59b306d078fb5bae9 2024-11-21T17:43:24,799 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/74e414eddab148b589b79f6e8c26b848 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74e414eddab148b589b79f6e8c26b848 2024-11-21T17:43:24,808 INFO [M:0;f3865e8353b4:46457 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/74e414eddab148b589b79f6e8c26b848, entries=8, sequenceid=317, filesize=5.5 K 2024-11-21T17:43:24,810 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/56e69f58722648d8819c0d93fd099a43 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/56e69f58722648d8819c0d93fd099a43 2024-11-21T17:43:24,818 INFO [M:0;f3865e8353b4:46457 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56e69f58722648d8819c0d93fd099a43 2024-11-21T17:43:24,818 INFO [M:0;f3865e8353b4:46457 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/56e69f58722648d8819c0d93fd099a43, entries=33, sequenceid=317, filesize=13.7 K 2024-11-21T17:43:24,819 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c4f03ee053340a59b306d078fb5bae9 as hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c4f03ee053340a59b306d078fb5bae9 2024-11-21T17:43:24,826 INFO [M:0;f3865e8353b4:46457 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37327/user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c4f03ee053340a59b306d078fb5bae9, entries=1, sequenceid=317, filesize=5.0 K 2024-11-21T17:43:24,828 INFO [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.98 KB/166891, heapSize ~190.57 KB/195144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1345ms, sequenceid=317, compaction requested=false 2024-11-21T17:43:24,829 INFO [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T17:43:24,829 DEBUG [M:0;f3865e8353b4:46457 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732211003481Disabling compacts and flushes for region at 1732211003481Disabling writes for close at 1732211003482 (+1 ms)Obtaining lock to block concurrent updates at 1732211003483 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732211003483Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166891, getHeapSize=195384, getOffHeapSize=0, getCellsCount=364 at 1732211003483Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732211003484 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732211003484Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732211003503 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732211003503Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732211003925 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732211003945 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732211003945Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732211004368 (+423 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732211004384 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732211004384Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d9764e0: reopening flushed file at 1732211004798 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@532beb98: reopening flushed file at 1732211004809 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@93d6cdf: reopening flushed file at 1732211004818 (+9 ms)Finished flush of dataSize ~162.98 KB/166891, heapSize ~190.57 KB/195144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1345ms, sequenceid=317, compaction requested=false at 1732211004828 (+10 ms)Writing region close event to WAL at 1732211004829 (+1 ms)Closed at 1732211004829 2024-11-21T17:43:24,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33819 is added to blk_1073741830_1006 (size=188103) 2024-11-21T17:43:24,832 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/33b3c74a-1060-2279-fdb0-d28bd500b808/MasterData/WALs/f3865e8353b4,46457,1732210897685/f3865e8353b4%2C46457%2C1732210897685.1732210900320 not finished, retry = 0 2024-11-21T17:43:24,934 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T17:43:24,934 INFO [M:0;f3865e8353b4:46457 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T17:43:24,934 INFO [M:0;f3865e8353b4:46457 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46457 2024-11-21T17:43:24,935 INFO [M:0;f3865e8353b4:46457 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T17:43:25,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T17:43:25,036 INFO [M:0;f3865e8353b4:46457 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T17:43:25,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46457-0x10063600c730000, quorum=127.0.0.1:56661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T17:43:25,047 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T17:43:25,050 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T17:43:25,050 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T17:43:25,050 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T17:43:25,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/hadoop.log.dir/,STOPPED} 2024-11-21T17:43:25,054 WARN [BP-330246287-172.17.0.2-1732210895193 heartbeating to localhost/127.0.0.1:37327 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T17:43:25,054 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T17:43:25,054 WARN [BP-330246287-172.17.0.2-1732210895193 heartbeating to localhost/127.0.0.1:37327 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-330246287-172.17.0.2-1732210895193 (Datanode Uuid 4a17df3d-ecb2-49d6-bbf2-3c24e42ae4a8) service to localhost/127.0.0.1:37327 2024-11-21T17:43:25,054 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T17:43:25,056 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127/data/data1/current/BP-330246287-172.17.0.2-1732210895193 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T17:43:25,056 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/cluster_506775a0-5bda-ebca-17f2-b327315ce127/data/data2/current/BP-330246287-172.17.0.2-1732210895193 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T17:43:25,056 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T17:43:25,067 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T17:43:25,068 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T17:43:25,068 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T17:43:25,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T17:43:25,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5020a721-4b2b-8277-1183-7bc042a188b9/hadoop.log.dir/,STOPPED} 2024-11-21T17:43:25,069 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T17:43:25,072 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-21T17:43:25,098 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T17:43:25,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down