2024-12-03 18:35:28,500 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 18:35:28,517 main DEBUG Took 0.013995 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 18:35:28,517 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 18:35:28,518 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 18:35:28,519 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 18:35:28,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,558 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 18:35:28,577 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,579 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,580 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,581 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,582 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,584 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,586 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,587 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,589 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,590 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,592 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,593 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,593 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,594 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:35:28,594 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,594 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 18:35:28,596 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:35:28,598 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 18:35:28,600 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 18:35:28,601 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 18:35:28,602 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 18:35:28,603 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 18:35:28,613 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 18:35:28,619 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 18:35:28,621 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 18:35:28,622 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 18:35:28,622 main DEBUG createAppenders(={Console}) 2024-12-03 18:35:28,623 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-03 18:35:28,624 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 18:35:28,624 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-03 18:35:28,625 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 18:35:28,625 main DEBUG OutputStream closed 2024-12-03 18:35:28,625 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 18:35:28,626 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 18:35:28,626 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-03 18:35:28,710 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 18:35:28,712 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 18:35:28,713 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 18:35:28,714 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 18:35:28,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 18:35:28,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 18:35:28,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 18:35:28,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 18:35:28,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 18:35:28,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 18:35:28,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 18:35:28,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 18:35:28,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 18:35:28,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 18:35:28,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 18:35:28,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 18:35:28,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 18:35:28,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 18:35:28,723 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 18:35:28,724 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-03 18:35:28,724 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 18:35:28,725 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-03T18:35:28,742 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-03 18:35:28,746 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 18:35:28,746 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T18:35:29,137 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89 2024-12-03T18:35:29,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T18:35:29,197 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722, deleteOnExit=true 2024-12-03T18:35:29,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T18:35:29,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/test.cache.data in system properties and HBase conf 2024-12-03T18:35:29,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T18:35:29,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/hadoop.log.dir in system properties and HBase conf 2024-12-03T18:35:29,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T18:35:29,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T18:35:29,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T18:35:29,363 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T18:35:29,492 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T18:35:29,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T18:35:29,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T18:35:29,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T18:35:29,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T18:35:29,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T18:35:29,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T18:35:29,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T18:35:29,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T18:35:29,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T18:35:29,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/nfs.dump.dir in system properties and HBase conf 2024-12-03T18:35:29,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/java.io.tmpdir in system properties and HBase conf 2024-12-03T18:35:29,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T18:35:29,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T18:35:29,505 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T18:35:30,844 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T18:35:30,977 INFO [Time-limited test {}] log.Log(170): Logging initialized @3510ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T18:35:31,084 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T18:35:31,181 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T18:35:31,220 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T18:35:31,222 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T18:35:31,224 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T18:35:31,243 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T18:35:31,251 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/hadoop.log.dir/,AVAILABLE} 2024-12-03T18:35:31,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T18:35:31,513 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/java.io.tmpdir/jetty-localhost-38069-hadoop-hdfs-3_4_1-tests_jar-_-any-2206215123525978852/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T18:35:31,532 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:38069} 2024-12-03T18:35:31,532 INFO [Time-limited test {}] server.Server(415): Started @4065ms 2024-12-03T18:35:32,143 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T18:35:32,152 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T18:35:32,153 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T18:35:32,153 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T18:35:32,154 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T18:35:32,154 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/hadoop.log.dir/,AVAILABLE} 2024-12-03T18:35:32,155 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T18:35:32,279 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/java.io.tmpdir/jetty-localhost-41793-hadoop-hdfs-3_4_1-tests_jar-_-any-12791230420324967831/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T18:35:32,280 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:41793} 2024-12-03T18:35:32,280 INFO [Time-limited test {}] server.Server(415): Started @4814ms 2024-12-03T18:35:32,353 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T18:35:33,660 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722/data/data1/current/BP-94149262-172.17.0.3-1733250930266/current, will proceed with Du for space computation calculation, 2024-12-03T18:35:33,663 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722/data/data2/current/BP-94149262-172.17.0.3-1733250930266/current, will proceed with Du for space computation calculation, 2024-12-03T18:35:33,762 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T18:35:33,855 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x60bf5fa11395c22f with lease ID 0xab3d86bce2adcfab: Processing first storage report for DS-2a70cff3-f107-4c00-ba49-3133a92156f2 from datanode DatanodeRegistration(127.0.0.1:36553, datanodeUuid=82d58ec3-17d6-499f-86cb-8427e1f3e0fb, infoPort=44663, infoSecurePort=0, ipcPort=45907, storageInfo=lv=-57;cid=testClusterID;nsid=418749612;c=1733250930266) 2024-12-03T18:35:33,856 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60bf5fa11395c22f with lease ID 0xab3d86bce2adcfab: from storage DS-2a70cff3-f107-4c00-ba49-3133a92156f2 node DatanodeRegistration(127.0.0.1:36553, datanodeUuid=82d58ec3-17d6-499f-86cb-8427e1f3e0fb, infoPort=44663, infoSecurePort=0, ipcPort=45907, storageInfo=lv=-57;cid=testClusterID;nsid=418749612;c=1733250930266), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T18:35:33,856 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x60bf5fa11395c22f with lease ID 0xab3d86bce2adcfab: Processing first storage report for DS-f987c2f8-5c30-4b4d-a2c6-456991d96ea6 from datanode DatanodeRegistration(127.0.0.1:36553, datanodeUuid=82d58ec3-17d6-499f-86cb-8427e1f3e0fb, infoPort=44663, infoSecurePort=0, ipcPort=45907, storageInfo=lv=-57;cid=testClusterID;nsid=418749612;c=1733250930266) 2024-12-03T18:35:33,857 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60bf5fa11395c22f with lease ID 0xab3d86bce2adcfab: from storage DS-f987c2f8-5c30-4b4d-a2c6-456991d96ea6 node DatanodeRegistration(127.0.0.1:36553, datanodeUuid=82d58ec3-17d6-499f-86cb-8427e1f3e0fb, infoPort=44663, infoSecurePort=0, ipcPort=45907, storageInfo=lv=-57;cid=testClusterID;nsid=418749612;c=1733250930266), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T18:35:33,946 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89 2024-12-03T18:35:34,153 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722/zookeeper_0, clientPort=53579, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T18:35:34,185 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53579 2024-12-03T18:35:34,235 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:35:34,241 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:35:34,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741825_1001 (size=7) 2024-12-03T18:35:35,215 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 with version=8 2024-12-03T18:35:35,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/hbase-staging 2024-12-03T18:35:35,347 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T18:35:35,702 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5cd6afc4bcac:0 server-side Connection retries=9 2024-12-03T18:35:35,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:35:35,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T18:35:35,727 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T18:35:35,727 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:35:35,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T18:35:35,974 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T18:35:36,072 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T18:35:36,085 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T18:35:36,089 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T18:35:36,127 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 91206 (auto-detected) 2024-12-03T18:35:36,129 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T18:35:36,164 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44487 2024-12-03T18:35:36,199 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44487 connecting to ZooKeeper ensemble=127.0.0.1:53579 2024-12-03T18:35:36,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444870x0, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T18:35:36,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44487-0x1019dc3f80f0000 connected 2024-12-03T18:35:36,600 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:35:36,604 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:35:36,637 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T18:35:36,643 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061, hbase.cluster.distributed=false 2024-12-03T18:35:36,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T18:35:36,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44487 2024-12-03T18:35:36,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44487 2024-12-03T18:35:36,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44487 2024-12-03T18:35:36,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44487 2024-12-03T18:35:36,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44487 2024-12-03T18:35:37,014 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5cd6afc4bcac:0 server-side Connection retries=9 2024-12-03T18:35:37,017 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:35:37,017 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T18:35:37,017 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T18:35:37,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:35:37,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T18:35:37,022 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T18:35:37,026 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T18:35:37,040 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34371 2024-12-03T18:35:37,042 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34371 connecting to ZooKeeper ensemble=127.0.0.1:53579 2024-12-03T18:35:37,044 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:35:37,049 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:35:37,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:343710x0, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T18:35:37,090 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34371-0x1019dc3f80f0001 connected 2024-12-03T18:35:37,090 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T18:35:37,097 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T18:35:37,128 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T18:35:37,138 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T18:35:37,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T18:35:37,150 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34371 2024-12-03T18:35:37,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34371 2024-12-03T18:35:37,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34371 2024-12-03T18:35:37,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34371 2024-12-03T18:35:37,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34371 2024-12-03T18:35:37,185 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5cd6afc4bcac:44487 2024-12-03T18:35:37,191 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5cd6afc4bcac,44487,1733250935430 2024-12-03T18:35:37,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:35:37,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:35:37,251 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5cd6afc4bcac,44487,1733250935430 2024-12-03T18:35:37,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T18:35:37,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:37,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:37,351 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T18:35:37,363 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5cd6afc4bcac,44487,1733250935430 from backup master directory 2024-12-03T18:35:37,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:35:37,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5cd6afc4bcac,44487,1733250935430 2024-12-03T18:35:37,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:35:37,403 WARN [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T18:35:37,404 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5cd6afc4bcac,44487,1733250935430 2024-12-03T18:35:37,406 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T18:35:37,418 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T18:35:37,534 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/hbase.id] with ID: e3beda95-f17d-4f45-8b3d-53cb24272a9e 2024-12-03T18:35:37,534 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/.tmp/hbase.id 2024-12-03T18:35:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741826_1002 (size=42) 2024-12-03T18:35:37,985 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/.tmp/hbase.id]:[hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/hbase.id] 2024-12-03T18:35:38,038 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:35:38,043 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T18:35:38,072 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-12-03T18:35:38,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:38,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:38,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741827_1003 (size=196) 2024-12-03T18:35:38,535 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:35:38,538 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T18:35:38,561 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T18:35:38,574 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T18:35:38,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741828_1004 (size=1189) 2024-12-03T18:35:39,048 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store 2024-12-03T18:35:39,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741829_1005 (size=34) 2024-12-03T18:35:39,077 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T18:35:39,082 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:35:39,083 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T18:35:39,084 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:35:39,084 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:35:39,086 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T18:35:39,086 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:35:39,086 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:35:39,088 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733250939083Disabling compacts and flushes for region at 1733250939083Disabling writes for close at 1733250939086 (+3 ms)Writing region close event to WAL at 1733250939086Closed at 1733250939086 2024-12-03T18:35:39,091 WARN [master/5cd6afc4bcac:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/.initializing 2024-12-03T18:35:39,091 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/WALs/5cd6afc4bcac,44487,1733250935430 2024-12-03T18:35:39,100 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T18:35:39,117 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5cd6afc4bcac%2C44487%2C1733250935430, suffix=, logDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/WALs/5cd6afc4bcac,44487,1733250935430, archiveDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/oldWALs, maxLogs=10 2024-12-03T18:35:39,147 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/WALs/5cd6afc4bcac,44487,1733250935430/5cd6afc4bcac%2C44487%2C1733250935430.1733250939122, exclude list is [], retry=0 2024-12-03T18:35:39,173 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36553,DS-2a70cff3-f107-4c00-ba49-3133a92156f2,DISK] 2024-12-03T18:35:39,178 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T18:35:39,243 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/WALs/5cd6afc4bcac,44487,1733250935430/5cd6afc4bcac%2C44487%2C1733250935430.1733250939122 2024-12-03T18:35:39,245 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44663:44663)] 2024-12-03T18:35:39,246 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:35:39,246 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:35:39,251 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,252 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T18:35:39,344 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:39,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:35:39,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T18:35:39,354 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:39,358 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:35:39,359 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,363 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T18:35:39,364 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:39,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:35:39,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T18:35:39,373 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:39,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:35:39,375 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,380 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,381 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,390 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,392 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,398 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T18:35:39,406 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:35:39,412 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:35:39,414 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60728343, jitterRate=-0.09507717192173004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T18:35:39,420 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733250939267Initializing all the Stores at 1733250939269 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733250939270 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250939270Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250939271 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250939271Cleaning up temporary data from old regions at 1733250939392 (+121 ms)Region opened successfully at 1733250939420 (+28 ms) 2024-12-03T18:35:39,421 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T18:35:39,474 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abcdba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5cd6afc4bcac/172.17.0.3:0 2024-12-03T18:35:39,551 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T18:35:39,568 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T18:35:39,568 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T18:35:39,573 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T18:35:39,587 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 13 msec 2024-12-03T18:35:39,597 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-12-03T18:35:39,597 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T18:35:39,634 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T18:35:39,646 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T18:35:39,679 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T18:35:39,682 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T18:35:39,685 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T18:35:39,695 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T18:35:39,698 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T18:35:39,703 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T18:35:39,712 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T18:35:39,715 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T18:35:39,720 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T18:35:39,743 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T18:35:39,753 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T18:35:39,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T18:35:39,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T18:35:39,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:39,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:39,772 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5cd6afc4bcac,44487,1733250935430, sessionid=0x1019dc3f80f0000, setting cluster-up flag (Was=false) 2024-12-03T18:35:39,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:39,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:39,820 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T18:35:39,825 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5cd6afc4bcac,44487,1733250935430 2024-12-03T18:35:39,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:39,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:35:39,870 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T18:35:39,872 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5cd6afc4bcac,44487,1733250935430 2024-12-03T18:35:39,880 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T18:35:39,945 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T18:35:39,953 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T18:35:39,959 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T18:35:39,964 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5cd6afc4bcac,44487,1733250935430 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T18:35:39,971 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5cd6afc4bcac:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:35:39,971 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5cd6afc4bcac:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:35:39,971 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5cd6afc4bcac:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:35:39,971 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5cd6afc4bcac:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:35:39,971 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5cd6afc4bcac:0, corePoolSize=10, maxPoolSize=10 2024-12-03T18:35:39,972 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:39,972 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5cd6afc4bcac:0, corePoolSize=2, maxPoolSize=2 2024-12-03T18:35:39,972 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:39,975 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(746): ClusterId : e3beda95-f17d-4f45-8b3d-53cb24272a9e 2024-12-03T18:35:39,975 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733250969975 2024-12-03T18:35:39,977 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T18:35:39,978 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T18:35:39,978 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T18:35:39,978 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T18:35:39,978 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T18:35:39,982 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T18:35:39,983 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T18:35:39,983 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T18:35:39,983 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T18:35:39,985 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:39,985 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T18:35:39,984 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:39,989 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T18:35:39,989 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T18:35:39,989 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T18:35:39,990 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T18:35:39,991 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T18:35:39,995 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T18:35:39,995 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T18:35:39,996 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T18:35:39,997 DEBUG [RS:0;5cd6afc4bcac:34371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39febd0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5cd6afc4bcac/172.17.0.3:0 2024-12-03T18:35:39,999 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5cd6afc4bcac:0:becomeActiveMaster-HFileCleaner.large.0-1733250939996,5,FailOnTimeoutGroup] 2024-12-03T18:35:40,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741831_1007 (size=1321) 2024-12-03T18:35:40,003 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5cd6afc4bcac:0:becomeActiveMaster-HFileCleaner.small.0-1733250939999,5,FailOnTimeoutGroup] 2024-12-03T18:35:40,003 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,003 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T18:35:40,004 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,005 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,014 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5cd6afc4bcac:34371 2024-12-03T18:35:40,017 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T18:35:40,018 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T18:35:40,018 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T18:35:40,021 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(2659): reportForDuty to master=5cd6afc4bcac,44487,1733250935430 with port=34371, startcode=1733250936949 2024-12-03T18:35:40,035 DEBUG [RS:0;5cd6afc4bcac:34371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T18:35:40,076 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59407, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T18:35:40,084 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5cd6afc4bcac,34371,1733250936949 2024-12-03T18:35:40,087 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(517): Registering regionserver=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:35:40,102 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:35:40,102 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37123 2024-12-03T18:35:40,102 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T18:35:40,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T18:35:40,121 DEBUG [RS:0;5cd6afc4bcac:34371 {}] zookeeper.ZKUtil(111): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5cd6afc4bcac,34371,1733250936949 2024-12-03T18:35:40,121 WARN [RS:0;5cd6afc4bcac:34371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T18:35:40,121 INFO [RS:0;5cd6afc4bcac:34371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T18:35:40,121 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949 2024-12-03T18:35:40,124 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5cd6afc4bcac,34371,1733250936949] 2024-12-03T18:35:40,168 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T18:35:40,183 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T18:35:40,195 INFO [RS:0;5cd6afc4bcac:34371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T18:35:40,195 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,197 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T18:35:40,204 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T18:35:40,206 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,206 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,206 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,206 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,207 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,207 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,207 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5cd6afc4bcac:0, corePoolSize=2, maxPoolSize=2 2024-12-03T18:35:40,207 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,207 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,207 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,208 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,208 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,208 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5cd6afc4bcac:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:35:40,208 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5cd6afc4bcac:0, corePoolSize=3, maxPoolSize=3 2024-12-03T18:35:40,208 DEBUG [RS:0;5cd6afc4bcac:34371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5cd6afc4bcac:0, corePoolSize=3, maxPoolSize=3 2024-12-03T18:35:40,210 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,211 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,211 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,211 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,212 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,212 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,34371,1733250936949-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T18:35:40,237 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T18:35:40,240 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,34371,1733250936949-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,240 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,240 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.Replication(171): 5cd6afc4bcac,34371,1733250936949 started 2024-12-03T18:35:40,263 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:35:40,264 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1482): Serving as 5cd6afc4bcac,34371,1733250936949, RpcServer on 5cd6afc4bcac/172.17.0.3:34371, sessionid=0x1019dc3f80f0001 2024-12-03T18:35:40,265 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T18:35:40,266 DEBUG [RS:0;5cd6afc4bcac:34371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5cd6afc4bcac,34371,1733250936949 2024-12-03T18:35:40,266 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5cd6afc4bcac,34371,1733250936949' 2024-12-03T18:35:40,266 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T18:35:40,273 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T18:35:40,275 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T18:35:40,275 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T18:35:40,275 DEBUG [RS:0;5cd6afc4bcac:34371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5cd6afc4bcac,34371,1733250936949 2024-12-03T18:35:40,276 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5cd6afc4bcac,34371,1733250936949' 2024-12-03T18:35:40,276 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T18:35:40,277 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T18:35:40,278 DEBUG [RS:0;5cd6afc4bcac:34371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T18:35:40,278 INFO [RS:0;5cd6afc4bcac:34371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T18:35:40,278 INFO [RS:0;5cd6afc4bcac:34371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T18:35:40,403 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T18:35:40,404 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:35:40,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741832_1008 (size=32) 2024-12-03T18:35:40,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:35:40,830 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T18:35:40,832 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T18:35:40,833 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:40,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:35:40,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T18:35:40,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T18:35:40,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:40,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:35:40,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T18:35:40,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T18:35:40,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:40,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:35:40,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T18:35:40,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T18:35:40,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:35:40,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:35:40,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T18:35:40,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740 2024-12-03T18:35:40,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740 2024-12-03T18:35:40,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T18:35:40,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T18:35:40,852 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T18:35:40,855 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T18:35:40,859 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:35:40,860 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69271939, jitterRate=0.03223232924938202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T18:35:40,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733250940825Initializing all the Stores at 1733250940829 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733250940829Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733250940830 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250940830Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733250940830Cleaning up temporary data from old regions at 1733250940851 (+21 ms)Region opened successfully at 1733250940862 (+11 ms) 2024-12-03T18:35:40,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T18:35:40,862 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T18:35:40,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T18:35:40,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T18:35:40,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T18:35:40,864 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T18:35:40,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733250940862Disabling compacts and flushes for region at 1733250940862Disabling writes for close at 1733250940863 (+1 ms)Writing region close event to WAL at 1733250940864 (+1 ms)Closed at 1733250940864 2024-12-03T18:35:40,867 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T18:35:40,867 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T18:35:40,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T18:35:40,879 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T18:35:40,882 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T18:35:41,033 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:42,034 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:43,034 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:44,035 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:45,036 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:46,037 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:46,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T18:35:46,066 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T18:35:47,037 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:48,038 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:49,039 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:50,040 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:51,041 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:52,041 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:53,042 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:54,043 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:55,043 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:56,044 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:57,045 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:58,045 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:35:59,046 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:00,047 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:01,048 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:02,049 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:03,050 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:03,938 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T18:36:04,050 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:05,051 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:06,052 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:07,053 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:08,053 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:09,054 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:10,055 WARN [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:36:10,285 INFO [RS:0;5cd6afc4bcac:34371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T18:36:10,289 INFO [RS:0;5cd6afc4bcac:34371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5cd6afc4bcac%2C34371%2C1733250936949, suffix=, logDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949, archiveDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/oldWALs, maxLogs=32 2024-12-03T18:36:10,313 DEBUG [RS:0;5cd6afc4bcac:34371 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949/5cd6afc4bcac%2C34371%2C1733250936949.1733250970292, exclude list is [], retry=0 2024-12-03T18:36:10,318 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36553,DS-2a70cff3-f107-4c00-ba49-3133a92156f2,DISK] 2024-12-03T18:36:10,322 INFO [RS:0;5cd6afc4bcac:34371 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949/5cd6afc4bcac%2C34371%2C1733250936949.1733250970292 2024-12-03T18:36:10,323 DEBUG [RS:0;5cd6afc4bcac:34371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44663:44663)] 2024-12-03T18:36:10,557 DEBUG [5cd6afc4bcac:44487 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T18:36:10,570 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:10,577 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5cd6afc4bcac,34371,1733250936949, state=OPENING 2024-12-03T18:36:10,621 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T18:36:10,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:36:10,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:36:10,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:36:10,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:36:10,633 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T18:36:10,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:10,809 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T18:36:10,813 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59929, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T18:36:10,824 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T18:36:10,824 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T18:36:10,825 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T18:36:10,828 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5cd6afc4bcac%2C34371%2C1733250936949.meta, suffix=.meta, logDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949, archiveDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/oldWALs, maxLogs=32 2024-12-03T18:36:10,843 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949/5cd6afc4bcac%2C34371%2C1733250936949.meta.1733250970830.meta, exclude list is [], retry=0 2024-12-03T18:36:10,847 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36553,DS-2a70cff3-f107-4c00-ba49-3133a92156f2,DISK] 2024-12-03T18:36:10,850 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949/5cd6afc4bcac%2C34371%2C1733250936949.meta.1733250970830.meta 2024-12-03T18:36:10,852 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44663:44663)] 2024-12-03T18:36:10,852 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:10,854 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T18:36:10,856 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T18:36:10,861 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T18:36:10,866 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T18:36:10,868 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:10,868 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T18:36:10,868 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T18:36:10,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T18:36:10,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T18:36:10,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:10,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:36:10,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T18:36:10,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T18:36:10,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:10,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:36:10,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T18:36:10,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T18:36:10,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:10,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:36:10,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T18:36:10,884 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T18:36:10,884 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:10,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:36:10,885 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T18:36:10,890 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740 2024-12-03T18:36:10,893 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740 2024-12-03T18:36:10,896 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T18:36:10,896 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T18:36:10,897 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T18:36:10,900 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T18:36:10,902 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64391749, jitterRate=-0.040488168597221375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T18:36:10,902 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T18:36:10,904 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733250970868Writing region info on filesystem at 1733250970869 (+1 ms)Initializing all the Stores at 1733250970870 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733250970871 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733250970871Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250970871Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733250970871Cleaning up temporary data from old regions at 1733250970896 (+25 ms)Running coprocessor post-open hooks at 1733250970902 (+6 ms)Region opened successfully at 1733250970903 (+1 ms) 2024-12-03T18:36:10,910 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733250970802 2024-12-03T18:36:10,922 DEBUG [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T18:36:10,923 INFO [RS_OPEN_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T18:36:10,925 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:10,927 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5cd6afc4bcac,34371,1733250936949, state=OPEN 2024-12-03T18:36:10,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T18:36:10,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T18:36:10,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:36:10,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:36:10,992 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:11,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T18:36:11,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5cd6afc4bcac,34371,1733250936949 in 358 msec 2024-12-03T18:36:11,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T18:36:11,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.1310 sec 2024-12-03T18:36:11,010 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T18:36:11,010 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T18:36:11,028 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T18:36:11,029 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5cd6afc4bcac,34371,1733250936949, seqNum=-1] 2024-12-03T18:36:11,048 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T18:36:11,050 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T18:36:11,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.1610 sec 2024-12-03T18:36:11,071 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733250971071, completionTime=-1 2024-12-03T18:36:11,074 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T18:36:11,074 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T18:36:11,101 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T18:36:11,101 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733251031101 2024-12-03T18:36:11,101 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733251091101 2024-12-03T18:36:11,101 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-03T18:36:11,103 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,44487,1733250935430-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:36:11,103 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,44487,1733250935430-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:36:11,103 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,44487,1733250935430-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:36:11,105 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5cd6afc4bcac:44487, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:36:11,105 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T18:36:11,105 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T18:36:11,111 DEBUG [master/5cd6afc4bcac:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T18:36:11,131 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.727sec 2024-12-03T18:36:11,132 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T18:36:11,134 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T18:36:11,135 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T18:36:11,135 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T18:36:11,135 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T18:36:11,136 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,44487,1733250935430-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T18:36:11,136 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,44487,1733250935430-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T18:36:11,144 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T18:36:11,144 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T18:36:11,145 INFO [master/5cd6afc4bcac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5cd6afc4bcac,44487,1733250935430-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:36:11,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77804624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:36:11,238 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T18:36:11,238 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T18:36:11,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5cd6afc4bcac,44487,-1 for getting cluster id 2024-12-03T18:36:11,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T18:36:11,259 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e3beda95-f17d-4f45-8b3d-53cb24272a9e' 2024-12-03T18:36:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T18:36:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e3beda95-f17d-4f45-8b3d-53cb24272a9e" 2024-12-03T18:36:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b0ddee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:36:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5cd6afc4bcac,44487,-1] 2024-12-03T18:36:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T18:36:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:36:11,274 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T18:36:11,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fad6539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:36:11,280 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T18:36:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5cd6afc4bcac,34371,1733250936949, seqNum=-1] 2024-12-03T18:36:11,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T18:36:11,292 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T18:36:11,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5cd6afc4bcac,44487,1733250935430 2024-12-03T18:36:11,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68214a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:36:11,322 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5cd6afc4bcac,44487,-1 for getting cluster id 2024-12-03T18:36:11,322 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T18:36:11,324 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e3beda95-f17d-4f45-8b3d-53cb24272a9e' 2024-12-03T18:36:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T18:36:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e3beda95-f17d-4f45-8b3d-53cb24272a9e" 2024-12-03T18:36:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c4d5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:36:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5cd6afc4bcac,44487,-1] 2024-12-03T18:36:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T18:36:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:36:11,330 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T18:36:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666749f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:36:11,348 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=213, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=636, ProcessCount=11, AvailableMemoryMB=3221 2024-12-03T18:36:11,356 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T18:36:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5cd6afc4bcac,44487,1733250935430 2024-12-03T18:36:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e14daea 2024-12-03T18:36:11,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T18:36:11,368 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T18:36:11,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T18:36:11,390 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:11,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-03T18:36:11,393 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:11,396 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T18:36:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741835_1011 (size=428) 2024-12-03T18:36:11,812 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 418b0b5898107dd4564a6a590a4f6738, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:11,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741836_1012 (size=79) 2024-12-03T18:36:12,223 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:12,223 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 418b0b5898107dd4564a6a590a4f6738, disabling compactions & flushes 2024-12-03T18:36:12,223 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:36:12,223 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:36:12,224 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. after waiting 1 ms 2024-12-03T18:36:12,224 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:36:12,224 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:36:12,224 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 418b0b5898107dd4564a6a590a4f6738: Waiting for close lock at 1733250972223Disabling compacts and flushes for region at 1733250972223Disabling writes for close at 1733250972224 (+1 ms)Writing region close event to WAL at 1733250972224Closed at 1733250972224 2024-12-03T18:36:12,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:12,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733250972226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733250972226"}]},"ts":"1733250972226"} 2024-12-03T18:36:12,235 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:12,238 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:12,240 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250972238"}]},"ts":"1733250972238"} 2024-12-03T18:36:12,244 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-03T18:36:12,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=418b0b5898107dd4564a6a590a4f6738, ASSIGN}] 2024-12-03T18:36:12,249 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=418b0b5898107dd4564a6a590a4f6738, ASSIGN 2024-12-03T18:36:12,251 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=418b0b5898107dd4564a6a590a4f6738, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:12,402 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=418b0b5898107dd4564a6a590a4f6738, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:12,408 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=418b0b5898107dd4564a6a590a4f6738, ASSIGN because future has completed 2024-12-03T18:36:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T18:36:12,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 418b0b5898107dd4564a6a590a4f6738, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:12,570 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:36:12,571 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 418b0b5898107dd4564a6a590a4f6738, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:12,571 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,572 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:12,572 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,572 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,575 INFO [StoreOpener-418b0b5898107dd4564a6a590a4f6738-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,578 INFO [StoreOpener-418b0b5898107dd4564a6a590a4f6738-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 418b0b5898107dd4564a6a590a4f6738 columnFamilyName testFamily 2024-12-03T18:36:12,578 DEBUG [StoreOpener-418b0b5898107dd4564a6a590a4f6738-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:12,579 INFO [StoreOpener-418b0b5898107dd4564a6a590a4f6738-1 {}] regionserver.HStore(327): Store=418b0b5898107dd4564a6a590a4f6738/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:12,579 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,581 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,582 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,582 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,583 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,587 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,596 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:12,597 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 418b0b5898107dd4564a6a590a4f6738; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65815219, jitterRate=-0.01927681267261505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:12,597 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:36:12,599 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 418b0b5898107dd4564a6a590a4f6738: Running coprocessor pre-open hook at 1733250972572Writing region info on filesystem at 1733250972572Initializing all the Stores at 1733250972574 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250972575 (+1 ms)Cleaning up temporary data from old regions at 1733250972583 (+8 ms)Running coprocessor post-open hooks at 1733250972597 (+14 ms)Region opened successfully at 1733250972599 (+2 ms) 2024-12-03T18:36:12,601 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738., pid=6, masterSystemTime=1733250972563 2024-12-03T18:36:12,605 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:36:12,605 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:36:12,606 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=418b0b5898107dd4564a6a590a4f6738, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:12,610 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 418b0b5898107dd4564a6a590a4f6738, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:12,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T18:36:12,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 418b0b5898107dd4564a6a590a4f6738, server=5cd6afc4bcac,34371,1733250936949 in 203 msec 2024-12-03T18:36:12,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T18:36:12,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=418b0b5898107dd4564a6a590a4f6738, ASSIGN in 371 msec 2024-12-03T18:36:12,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:12,624 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250972624"}]},"ts":"1733250972624"} 2024-12-03T18:36:12,632 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-03T18:36:12,634 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:12,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2560 sec 2024-12-03T18:36:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T18:36:14,419 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-03T18:36:14,419 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-03T18:36:14,420 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:14,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-03T18:36:14,426 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:14,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-03T18:36:14,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T18:36:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5cd6afc4bcac,34371,1733250936949, seqNum=-1] 2024-12-03T18:36:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T18:36:14,433 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T18:36:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:14,454 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:14,458 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-03T18:36:14,462 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T18:36:14,467 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T18:36:14,469 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T18:36:14,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=574182955052695560 2024-12-03T18:36:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250977469, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:14,498 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T18:36:14,500 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T18:36:14,501 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:14,503 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:14,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=574182955052695562 2024-12-03T18:36:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250977503, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:14,509 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:14,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=574182955052695563 2024-12-03T18:36:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250977509, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:14,516 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:14,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=574182955052695564 2024-12-03T18:36:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250977515, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:14,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:14,535 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=219 (was 213) Potentially hanging thread: RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1184483284_22 at /127.0.0.1:36160 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=636 (was 636), ProcessCount=11 (was 11), AvailableMemoryMB=2895 (was 3221) 2024-12-03T18:36:14,544 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=219, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=636, ProcessCount=11, AvailableMemoryMB=2895 2024-12-03T18:36:14,548 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:14,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-03T18:36:14,554 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:14,555 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:14,555 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-03T18:36:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T18:36:14,557 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:14,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741837_1013 (size=414) 2024-12-03T18:36:14,974 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0195071d4b74ea008bb625bd3316e78, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:14,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741838_1014 (size=65) 2024-12-03T18:36:15,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:15,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing c0195071d4b74ea008bb625bd3316e78, disabling compactions & flushes 2024-12-03T18:36:15,386 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:36:15,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:36:15,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. after waiting 0 ms 2024-12-03T18:36:15,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:36:15,386 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:36:15,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0195071d4b74ea008bb625bd3316e78: Waiting for close lock at 1733250975386Disabling compacts and flushes for region at 1733250975386Disabling writes for close at 1733250975386Writing region close event to WAL at 1733250975386Closed at 1733250975386 2024-12-03T18:36:15,389 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:15,389 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733250975389"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733250975389"}]},"ts":"1733250975389"} 2024-12-03T18:36:15,394 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:15,396 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:15,397 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250975397"}]},"ts":"1733250975397"} 2024-12-03T18:36:15,401 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T18:36:15,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=c0195071d4b74ea008bb625bd3316e78, ASSIGN}] 2024-12-03T18:36:15,404 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=c0195071d4b74ea008bb625bd3316e78, ASSIGN 2024-12-03T18:36:15,407 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=c0195071d4b74ea008bb625bd3316e78, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:15,557 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c0195071d4b74ea008bb625bd3316e78, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:15,562 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=c0195071d4b74ea008bb625bd3316e78, ASSIGN because future has completed 2024-12-03T18:36:15,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0195071d4b74ea008bb625bd3316e78, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T18:36:15,729 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:36:15,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c0195071d4b74ea008bb625bd3316e78, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:15,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:15,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,732 INFO [StoreOpener-c0195071d4b74ea008bb625bd3316e78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,736 INFO [StoreOpener-c0195071d4b74ea008bb625bd3316e78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0195071d4b74ea008bb625bd3316e78 columnFamilyName testFamily 2024-12-03T18:36:15,736 DEBUG [StoreOpener-c0195071d4b74ea008bb625bd3316e78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:15,737 INFO [StoreOpener-c0195071d4b74ea008bb625bd3316e78-1 {}] regionserver.HStore(327): Store=c0195071d4b74ea008bb625bd3316e78/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:15,737 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,738 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,739 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:15,747 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened c0195071d4b74ea008bb625bd3316e78; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74350232, jitterRate=0.10790479183197021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:15,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:36:15,748 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c0195071d4b74ea008bb625bd3316e78: Running coprocessor pre-open hook at 1733250975730Writing region info on filesystem at 1733250975730Initializing all the Stores at 1733250975732 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250975732Cleaning up temporary data from old regions at 1733250975740 (+8 ms)Running coprocessor post-open hooks at 1733250975747 (+7 ms)Region opened successfully at 1733250975748 (+1 ms) 2024-12-03T18:36:15,749 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78., pid=9, masterSystemTime=1733250975717 2024-12-03T18:36:15,752 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:36:15,752 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:36:15,753 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c0195071d4b74ea008bb625bd3316e78, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:15,756 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0195071d4b74ea008bb625bd3316e78, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:15,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-03T18:36:15,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c0195071d4b74ea008bb625bd3316e78, server=5cd6afc4bcac,34371,1733250936949 in 196 msec 2024-12-03T18:36:15,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T18:36:15,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=c0195071d4b74ea008bb625bd3316e78, ASSIGN in 360 msec 2024-12-03T18:36:15,766 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:15,767 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250975766"}]},"ts":"1733250975766"} 2024-12-03T18:36:15,769 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T18:36:15,771 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:15,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2220 sec 2024-12-03T18:36:16,402 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:36:16,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T18:36:16,423 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:16,424 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T18:36:16,424 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T18:36:16,425 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T18:36:16,425 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:16,427 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T18:36:16,427 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:16,429 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-03T18:36:16,431 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T18:36:16,432 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-03T18:36:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T18:36:17,579 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-03T18:36:17,579 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T18:36:17,579 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:17,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T18:36:17,585 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:17,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-03T18:36:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:17,599 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:17,599 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T18:36:17,602 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T18:36:17,604 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T18:36:23,623 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=5cd6afc4bcac:34371 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6017ms, rpcTimeout=5999ms 2024-12-03T18:36:23,624 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6017ms, rpcTimeout=5999ms 2024-12-03T18:36:23,624 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 from cache 2024-12-03T18:36:23,633 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=222 (was 219) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=570 (was 636), ProcessCount=11 (was 11), AvailableMemoryMB=2625 (was 2895) 2024-12-03T18:36:23,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=570, ProcessCount=11, AvailableMemoryMB=2625 2024-12-03T18:36:23,643 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:23,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-03T18:36:23,647 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:23,647 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:23,647 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-03T18:36:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T18:36:23,649 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:23,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741839_1015 (size=412) 2024-12-03T18:36:24,063 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eddb370b264893212cc95f0f0df239d4, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:24,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741840_1016 (size=63) 2024-12-03T18:36:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-03T18:36:24,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:24,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing eddb370b264893212cc95f0f0df239d4, disabling compactions & flushes 2024-12-03T18:36:24,493 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:36:24,494 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:36:24,494 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. after waiting 0 ms 2024-12-03T18:36:24,494 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:36:24,494 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:36:24,494 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for eddb370b264893212cc95f0f0df239d4: Waiting for close lock at 1733250984493Disabling compacts and flushes for region at 1733250984493Disabling writes for close at 1733250984494 (+1 ms)Writing region close event to WAL at 1733250984494Closed at 1733250984494 2024-12-03T18:36:24,499 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:24,500 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733250984499"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733250984499"}]},"ts":"1733250984499"} 2024-12-03T18:36:24,504 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:24,506 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:24,506 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250984506"}]},"ts":"1733250984506"} 2024-12-03T18:36:24,510 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T18:36:24,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=eddb370b264893212cc95f0f0df239d4, ASSIGN}] 2024-12-03T18:36:24,513 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=eddb370b264893212cc95f0f0df239d4, ASSIGN 2024-12-03T18:36:24,515 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=eddb370b264893212cc95f0f0df239d4, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:24,666 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=eddb370b264893212cc95f0f0df239d4, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T18:36:24,675 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=eddb370b264893212cc95f0f0df239d4, ASSIGN because future has completed 2024-12-03T18:36:24,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure eddb370b264893212cc95f0f0df239d4, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:24,840 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:36:24,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => eddb370b264893212cc95f0f0df239d4, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:24,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:24,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,844 INFO [StoreOpener-eddb370b264893212cc95f0f0df239d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,847 INFO [StoreOpener-eddb370b264893212cc95f0f0df239d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eddb370b264893212cc95f0f0df239d4 columnFamilyName testFamily 2024-12-03T18:36:24,847 DEBUG [StoreOpener-eddb370b264893212cc95f0f0df239d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:24,848 INFO [StoreOpener-eddb370b264893212cc95f0f0df239d4-1 {}] regionserver.HStore(327): Store=eddb370b264893212cc95f0f0df239d4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:24,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,852 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,854 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:24,859 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened eddb370b264893212cc95f0f0df239d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71813761, jitterRate=0.07010842859745026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:24,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:36:24,860 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for eddb370b264893212cc95f0f0df239d4: Running coprocessor pre-open hook at 1733250984841Writing region info on filesystem at 1733250984841Initializing all the Stores at 1733250984843 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250984843Cleaning up temporary data from old regions at 1733250984852 (+9 ms)Running coprocessor post-open hooks at 1733250984859 (+7 ms)Region opened successfully at 1733250984860 (+1 ms) 2024-12-03T18:36:24,861 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4., pid=12, masterSystemTime=1733250984830 2024-12-03T18:36:24,864 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:36:24,864 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:36:24,865 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=eddb370b264893212cc95f0f0df239d4, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:24,868 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure eddb370b264893212cc95f0f0df239d4, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:24,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T18:36:24,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure eddb370b264893212cc95f0f0df239d4, server=5cd6afc4bcac,34371,1733250936949 in 195 msec 2024-12-03T18:36:24,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T18:36:24,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=eddb370b264893212cc95f0f0df239d4, ASSIGN in 364 msec 2024-12-03T18:36:24,878 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:24,878 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250984878"}]},"ts":"1733250984878"} 2024-12-03T18:36:24,881 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T18:36:24,882 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:24,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2390 sec 2024-12-03T18:36:26,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T18:36:26,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T18:36:26,700 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-03T18:36:26,700 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T18:36:26,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:26,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T18:36:26,707 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:26,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-03T18:36:26,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:26,722 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:26,722 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T18:36:26,726 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T18:36:26,728 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T18:36:26,729 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T18:36:26,730 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T18:36:28,434 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-03T18:36:30,368 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:36:32,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34371 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695573' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T18:36:32,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:41068 deadline: 1733250998735, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695573' does not exist 2024-12-03T18:36:32,740 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 5cd6afc4bcac,34371,1733250936949 for closing scanner id = 574182955052695573 for eddb370b264893212cc95f0f0df239d4 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695573' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T18:36:32,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=498 (was 570), ProcessCount=11 (was 11), AvailableMemoryMB=2471 (was 2625) 2024-12-03T18:36:32,763 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=498, ProcessCount=11, AvailableMemoryMB=2471 2024-12-03T18:36:32,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-03T18:36:32,771 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:32,771 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:32,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-03T18:36:32,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T18:36:32,773 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:32,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741841_1017 (size=423) 2024-12-03T18:36:33,187 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6b667c859f6fac4fb5357ca15c20aa4e, NAME => 'testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:33,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741842_1018 (size=74) 2024-12-03T18:36:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-03T18:36:33,600 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:33,601 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 6b667c859f6fac4fb5357ca15c20aa4e, disabling compactions & flushes 2024-12-03T18:36:33,601 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:36:33,601 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:36:33,601 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. after waiting 0 ms 2024-12-03T18:36:33,601 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:36:33,601 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:36:33,601 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6b667c859f6fac4fb5357ca15c20aa4e: Waiting for close lock at 1733250993601Disabling compacts and flushes for region at 1733250993601Disabling writes for close at 1733250993601Writing region close event to WAL at 1733250993601Closed at 1733250993601 2024-12-03T18:36:33,604 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:33,604 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733250993604"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733250993604"}]},"ts":"1733250993604"} 2024-12-03T18:36:33,608 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:33,610 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:33,611 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250993611"}]},"ts":"1733250993611"} 2024-12-03T18:36:33,614 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-03T18:36:33,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=6b667c859f6fac4fb5357ca15c20aa4e, ASSIGN}] 2024-12-03T18:36:33,616 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=6b667c859f6fac4fb5357ca15c20aa4e, ASSIGN 2024-12-03T18:36:33,618 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=6b667c859f6fac4fb5357ca15c20aa4e, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:33,768 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6b667c859f6fac4fb5357ca15c20aa4e, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:33,771 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=6b667c859f6fac4fb5357ca15c20aa4e, ASSIGN because future has completed 2024-12-03T18:36:33,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b667c859f6fac4fb5357ca15c20aa4e, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T18:36:33,933 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:36:33,933 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b667c859f6fac4fb5357ca15c20aa4e, NAME => 'testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:33,933 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,934 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:33,934 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,934 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,936 INFO [StoreOpener-6b667c859f6fac4fb5357ca15c20aa4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,938 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T18:36:33,940 INFO [StoreOpener-6b667c859f6fac4fb5357ca15c20aa4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b667c859f6fac4fb5357ca15c20aa4e columnFamilyName testFamily 2024-12-03T18:36:33,940 DEBUG [StoreOpener-6b667c859f6fac4fb5357ca15c20aa4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:33,941 INFO [StoreOpener-6b667c859f6fac4fb5357ca15c20aa4e-1 {}] regionserver.HStore(327): Store=6b667c859f6fac4fb5357ca15c20aa4e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:33,942 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,943 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,944 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,944 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,945 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,947 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,950 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:33,951 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 6b667c859f6fac4fb5357ca15c20aa4e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75258972, jitterRate=0.12144607305526733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:33,951 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:36:33,952 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 6b667c859f6fac4fb5357ca15c20aa4e: Running coprocessor pre-open hook at 1733250993934Writing region info on filesystem at 1733250993934Initializing all the Stores at 1733250993936 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250993936Cleaning up temporary data from old regions at 1733250993945 (+9 ms)Running coprocessor post-open hooks at 1733250993951 (+6 ms)Region opened successfully at 1733250993952 (+1 ms) 2024-12-03T18:36:33,953 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e., pid=15, masterSystemTime=1733250993925 2024-12-03T18:36:33,955 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:36:33,955 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:36:33,956 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6b667c859f6fac4fb5357ca15c20aa4e, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:33,959 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6b667c859f6fac4fb5357ca15c20aa4e, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:33,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T18:36:33,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 6b667c859f6fac4fb5357ca15c20aa4e, server=5cd6afc4bcac,34371,1733250936949 in 189 msec 2024-12-03T18:36:33,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T18:36:33,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=6b667c859f6fac4fb5357ca15c20aa4e, ASSIGN in 351 msec 2024-12-03T18:36:33,970 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:33,970 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250993970"}]},"ts":"1733250993970"} 2024-12-03T18:36:33,974 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-03T18:36:33,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:33,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2100 sec 2024-12-03T18:36:34,435 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-03T18:36:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T18:36:35,799 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-03T18:36:35,799 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-03T18:36:35,799 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:35,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-03T18:36:35,805 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:35,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-03T18:36:35,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:35,823 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:35,823 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-03T18:36:35,827 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T18:36:35,830 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T18:36:35,832 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T18:36:35,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=574182955052695577 2024-12-03T18:36:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250998832, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:35,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T18:36:35,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T18:36:35,842 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:35,846 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:35,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=574182955052695579 2024-12-03T18:36:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250998846, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:35,852 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:35,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=574182955052695580 2024-12-03T18:36:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250998852, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:35,858 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:35,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=574182955052695581 2024-12-03T18:36:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:41068 deadline: 1733250998857, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T18:36:35,863 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T18:36:35,873 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=222 (was 222), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 498), ProcessCount=11 (was 11), AvailableMemoryMB=2436 (was 2471) 2024-12-03T18:36:35,882 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=2436 2024-12-03T18:36:35,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-03T18:36:35,890 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:35,890 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:35,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-03T18:36:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T18:36:35,893 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:35,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741843_1019 (size=418) 2024-12-03T18:36:36,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T18:36:36,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:36,306 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed15c622b44212c73cc81573ddc49f28, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:36,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741844_1020 (size=69) 2024-12-03T18:36:36,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:36,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ed15c622b44212c73cc81573ddc49f28, disabling compactions & flushes 2024-12-03T18:36:36,727 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:36:36,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:36:36,727 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. after waiting 0 ms 2024-12-03T18:36:36,728 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:36:36,728 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:36:36,728 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed15c622b44212c73cc81573ddc49f28: Waiting for close lock at 1733250996727Disabling compacts and flushes for region at 1733250996727Disabling writes for close at 1733250996727Writing region close event to WAL at 1733250996728 (+1 ms)Closed at 1733250996728 2024-12-03T18:36:36,731 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:36,731 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733250996731"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733250996731"}]},"ts":"1733250996731"} 2024-12-03T18:36:36,736 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:36,738 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:36,738 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250996738"}]},"ts":"1733250996738"} 2024-12-03T18:36:36,742 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T18:36:36,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ed15c622b44212c73cc81573ddc49f28, ASSIGN}] 2024-12-03T18:36:36,745 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ed15c622b44212c73cc81573ddc49f28, ASSIGN 2024-12-03T18:36:36,748 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ed15c622b44212c73cc81573ddc49f28, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:36,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ed15c622b44212c73cc81573ddc49f28, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:36,903 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ed15c622b44212c73cc81573ddc49f28, ASSIGN because future has completed 2024-12-03T18:36:36,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed15c622b44212c73cc81573ddc49f28, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T18:36:37,066 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:36:37,067 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => ed15c622b44212c73cc81573ddc49f28, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:37,067 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,067 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:37,067 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,068 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,071 INFO [StoreOpener-ed15c622b44212c73cc81573ddc49f28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,075 INFO [StoreOpener-ed15c622b44212c73cc81573ddc49f28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed15c622b44212c73cc81573ddc49f28 columnFamilyName testFamily 2024-12-03T18:36:37,076 DEBUG [StoreOpener-ed15c622b44212c73cc81573ddc49f28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:37,077 INFO [StoreOpener-ed15c622b44212c73cc81573ddc49f28-1 {}] regionserver.HStore(327): Store=ed15c622b44212c73cc81573ddc49f28/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:37,077 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,079 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,080 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,081 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,081 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,083 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,092 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:37,093 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened ed15c622b44212c73cc81573ddc49f28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72619837, jitterRate=0.0821198970079422}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:37,093 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:36:37,094 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for ed15c622b44212c73cc81573ddc49f28: Running coprocessor pre-open hook at 1733250997068Writing region info on filesystem at 1733250997068Initializing all the Stores at 1733250997070 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733250997070Cleaning up temporary data from old regions at 1733250997081 (+11 ms)Running coprocessor post-open hooks at 1733250997093 (+12 ms)Region opened successfully at 1733250997094 (+1 ms) 2024-12-03T18:36:37,099 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., pid=18, masterSystemTime=1733250997059 2024-12-03T18:36:37,103 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:36:37,103 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:36:37,104 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ed15c622b44212c73cc81573ddc49f28, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:37,107 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed15c622b44212c73cc81573ddc49f28, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:37,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T18:36:37,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure ed15c622b44212c73cc81573ddc49f28, server=5cd6afc4bcac,34371,1733250936949 in 206 msec 2024-12-03T18:36:37,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T18:36:37,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ed15c622b44212c73cc81573ddc49f28, ASSIGN in 373 msec 2024-12-03T18:36:37,120 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:37,120 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733250997120"}]},"ts":"1733250997120"} 2024-12-03T18:36:37,122 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T18:36:37,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:37,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2390 sec 2024-12-03T18:36:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T18:36:38,928 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-03T18:36:38,928 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T18:36:38,929 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:38,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T18:36:38,934 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:38,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-03T18:36:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:38,950 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:38,950 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T18:36:38,953 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T18:36:38,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T18:36:39,437 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:36:39,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=5cd6afc4bcac:34371 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-03T18:36:39,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-03T18:36:39,961 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 from cache 2024-12-03T18:36:40,436 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-03T18:36:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-03T18:36:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:40,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T18:36:41,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=5cd6afc4bcac:34371 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=979ms, rpcTimeout=972ms 2024-12-03T18:36:41,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=979ms, rpcTimeout=972ms 2024-12-03T18:36:41,960 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 from cache 2024-12-03T18:36:41,969 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=221 (was 222), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=445 (was 466), ProcessCount=11 (was 11), AvailableMemoryMB=2753 (was 2436) - AvailableMemoryMB LEAK? - 2024-12-03T18:36:41,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=221, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=445, ProcessCount=11, AvailableMemoryMB=2752 2024-12-03T18:36:41,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-03T18:36:41,982 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:41,983 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:41,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-03T18:36:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T18:36:41,984 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:41,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741845_1021 (size=416) 2024-12-03T18:36:41,997 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f54ba82be14a86b5bcae0075aaf6d415, NAME => 'testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:42,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741846_1022 (size=67) 2024-12-03T18:36:42,009 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:42,009 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing f54ba82be14a86b5bcae0075aaf6d415, disabling compactions & flushes 2024-12-03T18:36:42,010 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:36:42,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:36:42,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. after waiting 0 ms 2024-12-03T18:36:42,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:36:42,010 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:36:42,010 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f54ba82be14a86b5bcae0075aaf6d415: Waiting for close lock at 1733251002009Disabling compacts and flushes for region at 1733251002009Disabling writes for close at 1733251002010 (+1 ms)Writing region close event to WAL at 1733251002010Closed at 1733251002010 2024-12-03T18:36:42,012 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:42,013 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733251002012"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733251002012"}]},"ts":"1733251002012"} 2024-12-03T18:36:42,016 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:42,017 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:42,018 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251002017"}]},"ts":"1733251002017"} 2024-12-03T18:36:42,021 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T18:36:42,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f54ba82be14a86b5bcae0075aaf6d415, ASSIGN}] 2024-12-03T18:36:42,023 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f54ba82be14a86b5bcae0075aaf6d415, ASSIGN 2024-12-03T18:36:42,025 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f54ba82be14a86b5bcae0075aaf6d415, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:42,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f54ba82be14a86b5bcae0075aaf6d415, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:42,179 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f54ba82be14a86b5bcae0075aaf6d415, ASSIGN because future has completed 2024-12-03T18:36:42,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f54ba82be14a86b5bcae0075aaf6d415, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:42,339 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:36:42,339 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => f54ba82be14a86b5bcae0075aaf6d415, NAME => 'testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:42,340 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,340 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:42,340 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,340 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,343 INFO [StoreOpener-f54ba82be14a86b5bcae0075aaf6d415-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,345 INFO [StoreOpener-f54ba82be14a86b5bcae0075aaf6d415-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f54ba82be14a86b5bcae0075aaf6d415 columnFamilyName testFamily 2024-12-03T18:36:42,345 DEBUG [StoreOpener-f54ba82be14a86b5bcae0075aaf6d415-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:42,346 INFO [StoreOpener-f54ba82be14a86b5bcae0075aaf6d415-1 {}] regionserver.HStore(327): Store=f54ba82be14a86b5bcae0075aaf6d415/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:42,346 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,348 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,348 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,349 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,349 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,352 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,355 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:42,356 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened f54ba82be14a86b5bcae0075aaf6d415; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74769886, jitterRate=0.11415812373161316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:42,356 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:36:42,357 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for f54ba82be14a86b5bcae0075aaf6d415: Running coprocessor pre-open hook at 1733251002340Writing region info on filesystem at 1733251002340Initializing all the Stores at 1733251002342 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733251002342Cleaning up temporary data from old regions at 1733251002349 (+7 ms)Running coprocessor post-open hooks at 1733251002356 (+7 ms)Region opened successfully at 1733251002357 (+1 ms) 2024-12-03T18:36:42,360 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415., pid=21, masterSystemTime=1733251002333 2024-12-03T18:36:42,363 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:36:42,364 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:36:42,365 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f54ba82be14a86b5bcae0075aaf6d415, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:42,367 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f54ba82be14a86b5bcae0075aaf6d415, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:42,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T18:36:42,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure f54ba82be14a86b5bcae0075aaf6d415, server=5cd6afc4bcac,34371,1733250936949 in 190 msec 2024-12-03T18:36:42,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T18:36:42,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f54ba82be14a86b5bcae0075aaf6d415, ASSIGN in 352 msec 2024-12-03T18:36:42,377 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:42,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251002377"}]},"ts":"1733251002377"} 2024-12-03T18:36:42,380 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T18:36:42,382 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:42,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 403 msec 2024-12-03T18:36:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-03T18:36:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T18:36:42,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-03T18:36:42,998 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T18:36:42,999 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:43,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T18:36:43,004 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:43,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-03T18:36:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:43,035 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:43,035 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T18:36:43,038 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T18:36:43,041 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T18:36:43,043 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T18:36:43,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T18:36:46,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34371 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695592' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T18:36:46,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:41068 deadline: 1733251007053, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695592' does not exist 2024-12-03T18:36:46,055 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 5cd6afc4bcac,34371,1733250936949 for closing scanner id = 574182955052695592 for f54ba82be14a86b5bcae0075aaf6d415 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695592' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T18:36:46,059 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 221) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1184483284_22 at /127.0.0.1:55646 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 445), ProcessCount=11 (was 11), AvailableMemoryMB=3310 (was 2752) - AvailableMemoryMB LEAK? - 2024-12-03T18:36:46,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T18:36:46,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:46,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T18:36:46,066 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:46,074 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=3295 2024-12-03T18:36:46,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-03T18:36:46,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:46,081 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:46,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-03T18:36:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T18:36:46,084 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:46,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741847_1023 (size=411) 2024-12-03T18:36:46,438 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-03T18:36:46,497 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fefe678ddc652bf05153cb98174e563a, NAME => 'testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:46,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741848_1024 (size=62) 2024-12-03T18:36:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-03T18:36:46,907 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:46,907 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing fefe678ddc652bf05153cb98174e563a, disabling compactions & flushes 2024-12-03T18:36:46,907 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:36:46,908 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:36:46,908 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. after waiting 0 ms 2024-12-03T18:36:46,908 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:36:46,908 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:36:46,908 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for fefe678ddc652bf05153cb98174e563a: Waiting for close lock at 1733251006907Disabling compacts and flushes for region at 1733251006907Disabling writes for close at 1733251006908 (+1 ms)Writing region close event to WAL at 1733251006908Closed at 1733251006908 2024-12-03T18:36:46,909 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:46,910 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733251006909"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733251006909"}]},"ts":"1733251006909"} 2024-12-03T18:36:46,914 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:46,915 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:46,916 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251006916"}]},"ts":"1733251006916"} 2024-12-03T18:36:46,919 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T18:36:46,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fefe678ddc652bf05153cb98174e563a, ASSIGN}] 2024-12-03T18:36:46,921 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fefe678ddc652bf05153cb98174e563a, ASSIGN 2024-12-03T18:36:46,923 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fefe678ddc652bf05153cb98174e563a, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:47,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=fefe678ddc652bf05153cb98174e563a, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:47,078 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fefe678ddc652bf05153cb98174e563a, ASSIGN because future has completed 2024-12-03T18:36:47,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure fefe678ddc652bf05153cb98174e563a, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T18:36:47,273 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:36:47,274 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => fefe678ddc652bf05153cb98174e563a, NAME => 'testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:47,275 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,275 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:47,275 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,275 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,287 INFO [StoreOpener-fefe678ddc652bf05153cb98174e563a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,299 INFO [StoreOpener-fefe678ddc652bf05153cb98174e563a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fefe678ddc652bf05153cb98174e563a columnFamilyName testFamily 2024-12-03T18:36:47,300 DEBUG [StoreOpener-fefe678ddc652bf05153cb98174e563a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:47,305 INFO [StoreOpener-fefe678ddc652bf05153cb98174e563a-1 {}] regionserver.HStore(327): Store=fefe678ddc652bf05153cb98174e563a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:47,306 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,309 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,310 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,311 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,311 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,316 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,367 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:47,371 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened fefe678ddc652bf05153cb98174e563a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60237678, jitterRate=-0.1023886501789093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:47,371 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:36:47,372 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for fefe678ddc652bf05153cb98174e563a: Running coprocessor pre-open hook at 1733251007275Writing region info on filesystem at 1733251007275Initializing all the Stores at 1733251007280 (+5 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733251007280Cleaning up temporary data from old regions at 1733251007311 (+31 ms)Running coprocessor post-open hooks at 1733251007371 (+60 ms)Region opened successfully at 1733251007372 (+1 ms) 2024-12-03T18:36:47,374 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a., pid=24, masterSystemTime=1733251007242 2024-12-03T18:36:47,387 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:36:47,387 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:36:47,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=fefe678ddc652bf05153cb98174e563a, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:47,397 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure fefe678ddc652bf05153cb98174e563a, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:47,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T18:36:47,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure fefe678ddc652bf05153cb98174e563a, server=5cd6afc4bcac,34371,1733250936949 in 321 msec 2024-12-03T18:36:47,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T18:36:47,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fefe678ddc652bf05153cb98174e563a, ASSIGN in 486 msec 2024-12-03T18:36:47,415 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:47,416 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251007415"}]},"ts":"1733251007415"} 2024-12-03T18:36:47,433 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T18:36:47,443 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:47,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.3660 sec 2024-12-03T18:36:47,847 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:36:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T18:36:49,119 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-03T18:36:49,119 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T18:36:49,119 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:49,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T18:36:49,126 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:49,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-03T18:36:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:49,148 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:49,149 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T18:36:49,153 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T18:36:49,157 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T18:36:49,159 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T18:36:49,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T18:36:51,146 INFO [master/5cd6afc4bcac:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T18:36:51,146 INFO [master/5cd6afc4bcac:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T18:36:52,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34371 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695596' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T18:36:52,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:41068 deadline: 1733251013161, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695596' does not exist 2024-12-03T18:36:52,166 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 5cd6afc4bcac,34371,1733250936949 for closing scanner id = 574182955052695596 for fefe678ddc652bf05153cb98174e563a of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695596' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T18:36:52,171 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=221 (was 222), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=463 (was 425) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2852 (was 3295) 2024-12-03T18:36:52,182 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=221, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=463, ProcessCount=11, AvailableMemoryMB=2849 2024-12-03T18:36:52,187 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:36:52,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T18:36:52,197 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:36:52,197 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:52,197 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-03T18:36:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T18:36:52,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:36:52,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741849_1025 (size=419) 2024-12-03T18:36:52,611 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 69c9b05ed64699ca6d63936f3e4cdfb2, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:36:52,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741850_1026 (size=70) 2024-12-03T18:36:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-03T18:36:52,776 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:36:52,801 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-03T18:36:53,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:53,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 69c9b05ed64699ca6d63936f3e4cdfb2, disabling compactions & flushes 2024-12-03T18:36:53,020 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:36:53,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:36:53,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. after waiting 0 ms 2024-12-03T18:36:53,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:36:53,020 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:36:53,020 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 69c9b05ed64699ca6d63936f3e4cdfb2: Waiting for close lock at 1733251013020Disabling compacts and flushes for region at 1733251013020Disabling writes for close at 1733251013020Writing region close event to WAL at 1733251013020Closed at 1733251013020 2024-12-03T18:36:53,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:36:53,022 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733251013022"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733251013022"}]},"ts":"1733251013022"} 2024-12-03T18:36:53,025 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:36:53,026 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:36:53,026 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251013026"}]},"ts":"1733251013026"} 2024-12-03T18:36:53,029 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T18:36:53,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=69c9b05ed64699ca6d63936f3e4cdfb2, ASSIGN}] 2024-12-03T18:36:53,031 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=69c9b05ed64699ca6d63936f3e4cdfb2, ASSIGN 2024-12-03T18:36:53,032 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=69c9b05ed64699ca6d63936f3e4cdfb2, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:36:53,183 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=69c9b05ed64699ca6d63936f3e4cdfb2, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:53,186 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=69c9b05ed64699ca6d63936f3e4cdfb2, ASSIGN because future has completed 2024-12-03T18:36:53,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69c9b05ed64699ca6d63936f3e4cdfb2, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:36:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T18:36:53,346 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:36:53,346 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 69c9b05ed64699ca6d63936f3e4cdfb2, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:36:53,346 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,346 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:36:53,347 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,347 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,349 INFO [StoreOpener-69c9b05ed64699ca6d63936f3e4cdfb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,352 INFO [StoreOpener-69c9b05ed64699ca6d63936f3e4cdfb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69c9b05ed64699ca6d63936f3e4cdfb2 columnFamilyName testFamily 2024-12-03T18:36:53,352 DEBUG [StoreOpener-69c9b05ed64699ca6d63936f3e4cdfb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:36:53,353 INFO [StoreOpener-69c9b05ed64699ca6d63936f3e4cdfb2-1 {}] regionserver.HStore(327): Store=69c9b05ed64699ca6d63936f3e4cdfb2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:36:53,353 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,356 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,356 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,356 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,363 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:36:53,364 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 69c9b05ed64699ca6d63936f3e4cdfb2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66807445, jitterRate=-0.004491493105888367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:36:53,364 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:36:53,365 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 69c9b05ed64699ca6d63936f3e4cdfb2: Running coprocessor pre-open hook at 1733251013347Writing region info on filesystem at 1733251013347Initializing all the Stores at 1733251013349 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733251013349Cleaning up temporary data from old regions at 1733251013356 (+7 ms)Running coprocessor post-open hooks at 1733251013364 (+8 ms)Region opened successfully at 1733251013365 (+1 ms) 2024-12-03T18:36:53,367 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2., pid=27, masterSystemTime=1733251013338 2024-12-03T18:36:53,370 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:36:53,370 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:36:53,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=69c9b05ed64699ca6d63936f3e4cdfb2, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:36:53,376 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69c9b05ed64699ca6d63936f3e4cdfb2, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:36:53,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T18:36:53,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 69c9b05ed64699ca6d63936f3e4cdfb2, server=5cd6afc4bcac,34371,1733250936949 in 192 msec 2024-12-03T18:36:53,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T18:36:53,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=69c9b05ed64699ca6d63936f3e4cdfb2, ASSIGN in 353 msec 2024-12-03T18:36:53,388 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:36:53,389 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251013388"}]},"ts":"1733251013388"} 2024-12-03T18:36:53,392 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T18:36:53,393 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:36:53,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2060 sec 2024-12-03T18:36:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T18:36:55,220 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-03T18:36:55,220 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T18:36:55,221 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:55,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T18:36:55,227 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:36:55,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T18:36:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:36:55,256 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:36:55,256 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T18:36:55,262 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T18:36:55,265 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T18:36:56,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T18:36:56,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:56,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T18:36:56,066 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T18:36:58,848 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:36:58,871 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-03T18:37:01,270 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=5cd6afc4bcac:34371 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-12-03T18:37:01,270 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-12-03T18:37:01,270 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 from cache 2024-12-03T18:37:01,280 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=221 (was 221), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=484 (was 463) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2833 (was 2849) 2024-12-03T18:37:01,288 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=221, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=484, ProcessCount=11, AvailableMemoryMB=2832 2024-12-03T18:37:01,290 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:37:01,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-03T18:37:01,293 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:37:01,293 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:37:01,294 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-03T18:37:01,294 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:37:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T18:37:01,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741851_1027 (size=407) 2024-12-03T18:37:01,705 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a09aaf7de24e8ff87408593338defe9, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:37:01,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741852_1028 (size=58) 2024-12-03T18:37:01,718 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:37:01,718 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 1a09aaf7de24e8ff87408593338defe9, disabling compactions & flushes 2024-12-03T18:37:01,719 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:01,719 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:01,719 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. after waiting 0 ms 2024-12-03T18:37:01,719 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:01,719 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:01,719 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a09aaf7de24e8ff87408593338defe9: Waiting for close lock at 1733251021718Disabling compacts and flushes for region at 1733251021718Disabling writes for close at 1733251021719 (+1 ms)Writing region close event to WAL at 1733251021719Closed at 1733251021719 2024-12-03T18:37:01,721 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:37:01,721 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733251021721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733251021721"}]},"ts":"1733251021721"} 2024-12-03T18:37:01,727 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:37:01,729 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:37:01,729 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251021729"}]},"ts":"1733251021729"} 2024-12-03T18:37:01,733 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T18:37:01,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1a09aaf7de24e8ff87408593338defe9, ASSIGN}] 2024-12-03T18:37:01,737 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1a09aaf7de24e8ff87408593338defe9, ASSIGN 2024-12-03T18:37:01,743 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1a09aaf7de24e8ff87408593338defe9, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:37:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-03T18:37:01,894 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1a09aaf7de24e8ff87408593338defe9, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:37:01,896 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1a09aaf7de24e8ff87408593338defe9, ASSIGN because future has completed 2024-12-03T18:37:01,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a09aaf7de24e8ff87408593338defe9, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:37:02,055 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:02,055 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a09aaf7de24e8ff87408593338defe9, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:37:02,056 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,056 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:37:02,056 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,056 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,059 INFO [StoreOpener-1a09aaf7de24e8ff87408593338defe9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,061 INFO [StoreOpener-1a09aaf7de24e8ff87408593338defe9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a09aaf7de24e8ff87408593338defe9 columnFamilyName testFamily 2024-12-03T18:37:02,061 DEBUG [StoreOpener-1a09aaf7de24e8ff87408593338defe9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:37:02,062 INFO [StoreOpener-1a09aaf7de24e8ff87408593338defe9-1 {}] regionserver.HStore(327): Store=1a09aaf7de24e8ff87408593338defe9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:37:02,062 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,065 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,065 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,068 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,071 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:37:02,072 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 1a09aaf7de24e8ff87408593338defe9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67414332, jitterRate=0.004551827907562256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:37:02,073 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:02,073 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 1a09aaf7de24e8ff87408593338defe9: Running coprocessor pre-open hook at 1733251022057Writing region info on filesystem at 1733251022057Initializing all the Stores at 1733251022058 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733251022059 (+1 ms)Cleaning up temporary data from old regions at 1733251022065 (+6 ms)Running coprocessor post-open hooks at 1733251022073 (+8 ms)Region opened successfully at 1733251022073 2024-12-03T18:37:02,075 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9., pid=30, masterSystemTime=1733251022049 2024-12-03T18:37:02,077 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:02,078 INFO [RS_OPEN_PRIORITY_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:02,079 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1a09aaf7de24e8ff87408593338defe9, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:37:02,081 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a09aaf7de24e8ff87408593338defe9, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:37:02,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T18:37:02,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a09aaf7de24e8ff87408593338defe9, server=5cd6afc4bcac,34371,1733250936949 in 186 msec 2024-12-03T18:37:02,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T18:37:02,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1a09aaf7de24e8ff87408593338defe9, ASSIGN in 353 msec 2024-12-03T18:37:02,091 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:37:02,091 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251022091"}]},"ts":"1733251022091"} 2024-12-03T18:37:02,095 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T18:37:02,096 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:37:02,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 805 msec 2024-12-03T18:37:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T18:37:02,308 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-03T18:37:02,308 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T18:37:02,309 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:37:02,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T18:37:02,313 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:37:02,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-03T18:37:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:37:02,331 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:37:02,332 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T18:37:02,335 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T18:37:02,338 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T18:37:02,339 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T18:37:02,340 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T18:37:03,938 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T18:37:04,872 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-03T18:37:06,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T18:37:06,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T18:37:07,558 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:37:08,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34371 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695604' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T18:37:08,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:41068 deadline: 1733251034340, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695604' does not exist 2024-12-03T18:37:08,343 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 5cd6afc4bcac,34371,1733250936949 for closing scanner id = 574182955052695604 for 1a09aaf7de24e8ff87408593338defe9 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '574182955052695604' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T18:37:08,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=221 (was 221), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=461 (was 484), ProcessCount=11 (was 11), AvailableMemoryMB=2262 (was 2832) 2024-12-03T18:37:08,359 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=221, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=461, ProcessCount=11, AvailableMemoryMB=2262 2024-12-03T18:37:08,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:37:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T18:37:08,364 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:37:08,364 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:37:08,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-03T18:37:08,366 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:37:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T18:37:08,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741853_1029 (size=423) 2024-12-03T18:37:08,777 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45351b6e224a618af1394c5bb1fca4c0, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061 2024-12-03T18:37:08,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741854_1030 (size=74) 2024-12-03T18:37:08,793 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:37:08,793 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 45351b6e224a618af1394c5bb1fca4c0, disabling compactions & flushes 2024-12-03T18:37:08,794 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:08,794 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:08,794 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. after waiting 0 ms 2024-12-03T18:37:08,794 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:08,794 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:08,794 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45351b6e224a618af1394c5bb1fca4c0: Waiting for close lock at 1733251028793Disabling compacts and flushes for region at 1733251028793Disabling writes for close at 1733251028794 (+1 ms)Writing region close event to WAL at 1733251028794Closed at 1733251028794 2024-12-03T18:37:08,795 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:37:08,796 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733251028796"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733251028796"}]},"ts":"1733251028796"} 2024-12-03T18:37:08,799 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:37:08,800 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:37:08,800 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251028800"}]},"ts":"1733251028800"} 2024-12-03T18:37:08,803 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T18:37:08,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=45351b6e224a618af1394c5bb1fca4c0, ASSIGN}] 2024-12-03T18:37:08,805 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=45351b6e224a618af1394c5bb1fca4c0, ASSIGN 2024-12-03T18:37:08,806 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=45351b6e224a618af1394c5bb1fca4c0, ASSIGN; state=OFFLINE, location=5cd6afc4bcac,34371,1733250936949; forceNewPlan=false, retain=false 2024-12-03T18:37:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-03T18:37:08,957 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=45351b6e224a618af1394c5bb1fca4c0, regionState=OPENING, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:37:08,961 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=45351b6e224a618af1394c5bb1fca4c0, ASSIGN because future has completed 2024-12-03T18:37:08,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45351b6e224a618af1394c5bb1fca4c0, server=5cd6afc4bcac,34371,1733250936949}] 2024-12-03T18:37:09,122 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:09,122 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 45351b6e224a618af1394c5bb1fca4c0, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:37:09,122 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,123 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:37:09,123 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,123 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,125 INFO [StoreOpener-45351b6e224a618af1394c5bb1fca4c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,128 INFO [StoreOpener-45351b6e224a618af1394c5bb1fca4c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45351b6e224a618af1394c5bb1fca4c0 columnFamilyName testFamily 2024-12-03T18:37:09,128 DEBUG [StoreOpener-45351b6e224a618af1394c5bb1fca4c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:37:09,129 INFO [StoreOpener-45351b6e224a618af1394c5bb1fca4c0-1 {}] regionserver.HStore(327): Store=45351b6e224a618af1394c5bb1fca4c0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:37:09,129 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,131 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,132 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,132 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,133 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,140 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,153 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:37:09,155 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 45351b6e224a618af1394c5bb1fca4c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73962722, jitterRate=0.10213044285774231}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:37:09,155 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:09,156 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 45351b6e224a618af1394c5bb1fca4c0: Running coprocessor pre-open hook at 1733251029123Writing region info on filesystem at 1733251029123Initializing all the Stores at 1733251029125 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733251029125Cleaning up temporary data from old regions at 1733251029133 (+8 ms)Running coprocessor post-open hooks at 1733251029155 (+22 ms)Region opened successfully at 1733251029156 (+1 ms) 2024-12-03T18:37:09,157 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., pid=33, masterSystemTime=1733251029116 2024-12-03T18:37:09,164 DEBUG [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:09,164 INFO [RS_OPEN_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:09,171 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=45351b6e224a618af1394c5bb1fca4c0, regionState=OPEN, openSeqNum=2, regionLocation=5cd6afc4bcac,34371,1733250936949 2024-12-03T18:37:09,177 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45351b6e224a618af1394c5bb1fca4c0, server=5cd6afc4bcac,34371,1733250936949 because future has completed 2024-12-03T18:37:09,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T18:37:09,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 45351b6e224a618af1394c5bb1fca4c0, server=5cd6afc4bcac,34371,1733250936949 in 229 msec 2024-12-03T18:37:09,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T18:37:09,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=45351b6e224a618af1394c5bb1fca4c0, ASSIGN in 393 msec 2024-12-03T18:37:09,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:37:09,202 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733251029202"}]},"ts":"1733251029202"} 2024-12-03T18:37:09,213 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T18:37:09,215 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:37:09,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 854 msec 2024-12-03T18:37:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T18:37:09,378 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-03T18:37:09,378 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T18:37:09,378 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:37:09,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T18:37:09,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T18:37:09,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T18:37:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:37:09,402 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T18:37:09,402 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T18:37:09,406 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T18:37:09,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T18:37:10,410 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=5cd6afc4bcac:34371 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-03T18:37:10,410 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-03T18:37:10,411 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 from cache 2024-12-03T18:37:10,873 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-03T18:37:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-03T18:37:11,136 DEBUG [master/5cd6afc4bcac:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T18:37:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2] 2024-12-03T18:37:11,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T18:37:12,410 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=5cd6afc4bcac:34371 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=989ms, rpcTimeout=985ms 2024-12-03T18:37:12,410 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=989ms, rpcTimeout=985ms 2024-12-03T18:37:12,411 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., hostname=5cd6afc4bcac,34371,1733250936949, seqNum=2 from cache 2024-12-03T18:37:12,422 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 221) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1184483284_22 at /127.0.0.1:53178 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 461), ProcessCount=11 (was 11), AvailableMemoryMB=2158 (was 2262) 2024-12-03T18:37:12,423 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T18:37:12,423 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T18:37:12,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:37:12,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:37:12,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T18:37:12,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T18:37:12,430 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T18:37:12,430 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T18:37:12,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:37:12,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:37:12,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T18:37:12,430 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T18:37:12,430 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1871265054, stopped=false 2024-12-03T18:37:12,431 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5cd6afc4bcac,44487,1733250935430 2024-12-03T18:37:12,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T18:37:12,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T18:37:12,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:37:12,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:37:12,508 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T18:37:12,509 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T18:37:12,509 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T18:37:12,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:37:12,509 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T18:37:12,509 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5cd6afc4bcac,34371,1733250936949' ***** 2024-12-03T18:37:12,510 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T18:37:12,510 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T18:37:12,510 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T18:37:12,510 INFO [RS:0;5cd6afc4bcac:34371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T18:37:12,511 INFO [RS:0;5cd6afc4bcac:34371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T18:37:12,511 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for 45351b6e224a618af1394c5bb1fca4c0 2024-12-03T18:37:12,511 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for ed15c622b44212c73cc81573ddc49f28 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for fefe678ddc652bf05153cb98174e563a 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for c0195071d4b74ea008bb625bd3316e78 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for 69c9b05ed64699ca6d63936f3e4cdfb2 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for 6b667c859f6fac4fb5357ca15c20aa4e 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for eddb370b264893212cc95f0f0df239d4 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for 418b0b5898107dd4564a6a590a4f6738 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a09aaf7de24e8ff87408593338defe9 2024-12-03T18:37:12,512 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45351b6e224a618af1394c5bb1fca4c0, disabling compactions & flushes 2024-12-03T18:37:12,512 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:12,512 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:12,512 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:12,513 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(959): stopping server 5cd6afc4bcac,34371,1733250936949 2024-12-03T18:37:12,513 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. after waiting 0 ms 2024-12-03T18:37:12,513 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T18:37:12,513 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:12,513 INFO [RS:0;5cd6afc4bcac:34371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5cd6afc4bcac:34371. 2024-12-03T18:37:12,513 DEBUG [RS:0;5cd6afc4bcac:34371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T18:37:12,513 DEBUG [RS:0;5cd6afc4bcac:34371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:37:12,513 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T18:37:12,513 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T18:37:12,513 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T18:37:12,514 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T18:37:12,514 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-03T18:37:12,515 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T18:37:12,515 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T18:37:12,515 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T18:37:12,515 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T18:37:12,515 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T18:37:12,515 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 45351b6e224a618af1394c5bb1fca4c0, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, ed15c622b44212c73cc81573ddc49f28, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415, fefe678ddc652bf05153cb98174e563a 2024-12-03T18:37:12,518 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 45351b6e224a618af1394c5bb1fca4c0 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:12,518 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-03T18:37:12,575 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0/.tmp/testFamily/6610e6382911498d976e7de3cc4b9240 is 61, key is row-0/testFamily:testQualifier/1733251029391/Put/seqid=0 2024-12-03T18:37:12,579 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/info/9753c2775d034574821f1d3c7b866350 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738./info:regioninfo/1733250972606/Put/seqid=0 2024-12-03T18:37:12,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741855_1031 (size=5049) 2024-12-03T18:37:12,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741856_1032 (size=20564) 2024-12-03T18:37:12,715 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 45351b6e224a618af1394c5bb1fca4c0, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, ed15c622b44212c73cc81573ddc49f28, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415, fefe678ddc652bf05153cb98174e563a 2024-12-03T18:37:12,916 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 45351b6e224a618af1394c5bb1fca4c0, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, ed15c622b44212c73cc81573ddc49f28, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415, fefe678ddc652bf05153cb98174e563a 2024-12-03T18:37:12,923 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-03T18:37:12,987 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0/.tmp/testFamily/6610e6382911498d976e7de3cc4b9240 2024-12-03T18:37:12,987 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/info/9753c2775d034574821f1d3c7b866350 2024-12-03T18:37:13,040 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0/.tmp/testFamily/6610e6382911498d976e7de3cc4b9240 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0/testFamily/6610e6382911498d976e7de3cc4b9240 2024-12-03T18:37:13,051 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0/testFamily/6610e6382911498d976e7de3cc4b9240, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:13,057 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/ns/5835f108285248afa3c09bd68676090e is 43, key is default/ns:d/1733250971054/Put/seqid=0 2024-12-03T18:37:13,062 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 45351b6e224a618af1394c5bb1fca4c0 in 543ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:13,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741857_1033 (size=5153) 2024-12-03T18:37:13,069 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScannerAsync/45351b6e224a618af1394c5bb1fca4c0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:13,071 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:13,072 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45351b6e224a618af1394c5bb1fca4c0: Waiting for close lock at 1733251032512Running coprocessor pre-close hooks at 1733251032512Disabling compacts and flushes for region at 1733251032512Disabling writes for close at 1733251032513 (+1 ms)Obtaining lock to block concurrent updates at 1733251032518 (+5 ms)Preparing flush snapshotting stores in 45351b6e224a618af1394c5bb1fca4c0 at 1733251032518Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251032528 (+10 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. at 1733251032529 (+1 ms)Flushing 45351b6e224a618af1394c5bb1fca4c0/testFamily: creating writer at 1733251032531 (+2 ms)Flushing 45351b6e224a618af1394c5bb1fca4c0/testFamily: appending metadata at 1733251032569 (+38 ms)Flushing 45351b6e224a618af1394c5bb1fca4c0/testFamily: closing flushed file at 1733251032571 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73804b7a: reopening flushed file at 1733251033038 (+467 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 45351b6e224a618af1394c5bb1fca4c0 in 543ms, sequenceid=8, compaction requested=false at 1733251033062 (+24 ms)Writing region close event to WAL at 1733251033065 (+3 ms)Running coprocessor post-close hooks at 1733251033070 (+5 ms)Closed at 1733251033071 (+1 ms) 2024-12-03T18:37:13,072 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733251028361.45351b6e224a618af1394c5bb1fca4c0. 2024-12-03T18:37:13,072 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed15c622b44212c73cc81573ddc49f28, disabling compactions & flushes 2024-12-03T18:37:13,072 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:37:13,072 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:37:13,072 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. after waiting 0 ms 2024-12-03T18:37:13,072 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:37:13,072 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ed15c622b44212c73cc81573ddc49f28 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:13,091 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28/.tmp/testFamily/16d3bee77e5a4ae2aa68984438adc9d6 is 61, key is row-0/testFamily:testQualifier/1733250998939/Put/seqid=0 2024-12-03T18:37:13,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741858_1034 (size=5049) 2024-12-03T18:37:13,116 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, ed15c622b44212c73cc81573ddc49f28, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415, fefe678ddc652bf05153cb98174e563a 2024-12-03T18:37:13,211 INFO [regionserver/5cd6afc4bcac:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T18:37:13,211 INFO [regionserver/5cd6afc4bcac:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T18:37:13,316 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, ed15c622b44212c73cc81573ddc49f28, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415, fefe678ddc652bf05153cb98174e563a 2024-12-03T18:37:13,463 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/ns/5835f108285248afa3c09bd68676090e 2024-12-03T18:37:13,492 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/table/e23e46f6282c475c8f967d5e1068a313 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733250972624/Put/seqid=0 2024-12-03T18:37:13,497 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28/.tmp/testFamily/16d3bee77e5a4ae2aa68984438adc9d6 2024-12-03T18:37:13,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741859_1035 (size=6699) 2024-12-03T18:37:13,499 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/table/e23e46f6282c475c8f967d5e1068a313 2024-12-03T18:37:13,509 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28/.tmp/testFamily/16d3bee77e5a4ae2aa68984438adc9d6 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28/testFamily/16d3bee77e5a4ae2aa68984438adc9d6 2024-12-03T18:37:13,511 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/info/9753c2775d034574821f1d3c7b866350 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/info/9753c2775d034574821f1d3c7b866350 2024-12-03T18:37:13,516 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-03T18:37:13,517 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, ed15c622b44212c73cc81573ddc49f28, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415, fefe678ddc652bf05153cb98174e563a 2024-12-03T18:37:13,520 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/info/9753c2775d034574821f1d3c7b866350, entries=100, sequenceid=56, filesize=20.1 K 2024-12-03T18:37:13,522 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/ns/5835f108285248afa3c09bd68676090e as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/ns/5835f108285248afa3c09bd68676090e 2024-12-03T18:37:13,523 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28/testFamily/16d3bee77e5a4ae2aa68984438adc9d6, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:13,525 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ed15c622b44212c73cc81573ddc49f28 in 453ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:13,532 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnOpenScanner/ed15c622b44212c73cc81573ddc49f28/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:13,534 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:37:13,534 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed15c622b44212c73cc81573ddc49f28: Waiting for close lock at 1733251033072Running coprocessor pre-close hooks at 1733251033072Disabling compacts and flushes for region at 1733251033072Disabling writes for close at 1733251033072Obtaining lock to block concurrent updates at 1733251033072Preparing flush snapshotting stores in ed15c622b44212c73cc81573ddc49f28 at 1733251033072Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251033073 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. at 1733251033073Flushing ed15c622b44212c73cc81573ddc49f28/testFamily: creating writer at 1733251033073Flushing ed15c622b44212c73cc81573ddc49f28/testFamily: appending metadata at 1733251033090 (+17 ms)Flushing ed15c622b44212c73cc81573ddc49f28/testFamily: closing flushed file at 1733251033090Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35e78f64: reopening flushed file at 1733251033507 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ed15c622b44212c73cc81573ddc49f28 in 453ms, sequenceid=8, compaction requested=false at 1733251033525 (+18 ms)Writing region close event to WAL at 1733251033526 (+1 ms)Running coprocessor post-close hooks at 1733251033534 (+8 ms)Closed at 1733251033534 2024-12-03T18:37:13,535 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733250995885.ed15c622b44212c73cc81573ddc49f28. 2024-12-03T18:37:13,535 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/ns/5835f108285248afa3c09bd68676090e, entries=2, sequenceid=56, filesize=5.0 K 2024-12-03T18:37:13,535 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fefe678ddc652bf05153cb98174e563a, disabling compactions & flushes 2024-12-03T18:37:13,535 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:37:13,535 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:37:13,535 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. after waiting 0 ms 2024-12-03T18:37:13,535 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:37:13,535 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fefe678ddc652bf05153cb98174e563a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:13,538 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/.tmp/table/e23e46f6282c475c8f967d5e1068a313 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/table/e23e46f6282c475c8f967d5e1068a313 2024-12-03T18:37:13,551 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/table/e23e46f6282c475c8f967d5e1068a313, entries=20, sequenceid=56, filesize=6.5 K 2024-12-03T18:37:13,555 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1039ms, sequenceid=56, compaction requested=false 2024-12-03T18:37:13,560 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a/.tmp/testFamily/13dbf306b91a484ea066d56251b7ce0b is 61, key is row-0/testFamily:testQualifier/1733251009132/Put/seqid=0 2024-12-03T18:37:13,569 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-03T18:37:13,570 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T18:37:13,572 INFO [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T18:37:13,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741860_1036 (size=5049) 2024-12-03T18:37:13,572 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733251032514Running coprocessor pre-close hooks at 1733251032514Disabling compacts and flushes for region at 1733251032514Disabling writes for close at 1733251032515 (+1 ms)Obtaining lock to block concurrent updates at 1733251032518 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733251032518Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733251032529 (+11 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733251032529Flushing 1588230740/info: creating writer at 1733251032531 (+2 ms)Flushing 1588230740/info: appending metadata at 1733251032575 (+44 ms)Flushing 1588230740/info: closing flushed file at 1733251032575Flushing 1588230740/ns: creating writer at 1733251033038 (+463 ms)Flushing 1588230740/ns: appending metadata at 1733251033056 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733251033056Flushing 1588230740/table: creating writer at 1733251033473 (+417 ms)Flushing 1588230740/table: appending metadata at 1733251033491 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733251033491Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29ed1364: reopening flushed file at 1733251033509 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52325148: reopening flushed file at 1733251033520 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52dd6420: reopening flushed file at 1733251033535 (+15 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1039ms, sequenceid=56, compaction requested=false at 1733251033555 (+20 ms)Writing region close event to WAL at 1733251033561 (+6 ms)Running coprocessor post-close hooks at 1733251033570 (+9 ms)Closed at 1733251033572 (+2 ms) 2024-12-03T18:37:13,572 DEBUG [RS_CLOSE_META-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T18:37:13,576 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a/.tmp/testFamily/13dbf306b91a484ea066d56251b7ce0b 2024-12-03T18:37:13,586 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a/.tmp/testFamily/13dbf306b91a484ea066d56251b7ce0b as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a/testFamily/13dbf306b91a484ea066d56251b7ce0b 2024-12-03T18:37:13,596 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a/testFamily/13dbf306b91a484ea066d56251b7ce0b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:13,597 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fefe678ddc652bf05153cb98174e563a in 62ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:13,620 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNext/fefe678ddc652bf05153cb98174e563a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:13,621 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:37:13,622 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fefe678ddc652bf05153cb98174e563a: Waiting for close lock at 1733251033535Running coprocessor pre-close hooks at 1733251033535Disabling compacts and flushes for region at 1733251033535Disabling writes for close at 1733251033535Obtaining lock to block concurrent updates at 1733251033535Preparing flush snapshotting stores in fefe678ddc652bf05153cb98174e563a at 1733251033535Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251033535Flushing stores of testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. at 1733251033537 (+2 ms)Flushing fefe678ddc652bf05153cb98174e563a/testFamily: creating writer at 1733251033537Flushing fefe678ddc652bf05153cb98174e563a/testFamily: appending metadata at 1733251033560 (+23 ms)Flushing fefe678ddc652bf05153cb98174e563a/testFamily: closing flushed file at 1733251033560Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b28191f: reopening flushed file at 1733251033585 (+25 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fefe678ddc652bf05153cb98174e563a in 62ms, sequenceid=8, compaction requested=false at 1733251033597 (+12 ms)Writing region close event to WAL at 1733251033611 (+14 ms)Running coprocessor post-close hooks at 1733251033621 (+10 ms)Closed at 1733251033621 2024-12-03T18:37:13,622 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733251006076.fefe678ddc652bf05153cb98174e563a. 2024-12-03T18:37:13,622 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0195071d4b74ea008bb625bd3316e78, disabling compactions & flushes 2024-12-03T18:37:13,622 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:37:13,622 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:37:13,622 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. after waiting 0 ms 2024-12-03T18:37:13,623 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:37:13,623 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c0195071d4b74ea008bb625bd3316e78 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:13,647 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78/.tmp/testFamily/947ad9f375154097953d9462d2dbcead is 61, key is row-0/testFamily:testQualifier/1733250977590/Put/seqid=0 2024-12-03T18:37:13,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741861_1037 (size=5049) 2024-12-03T18:37:13,717 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:13,917 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, c0195071d4b74ea008bb625bd3316e78, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:14,054 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78/.tmp/testFamily/947ad9f375154097953d9462d2dbcead 2024-12-03T18:37:14,068 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78/.tmp/testFamily/947ad9f375154097953d9462d2dbcead as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78/testFamily/947ad9f375154097953d9462d2dbcead 2024-12-03T18:37:14,077 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78/testFamily/947ad9f375154097953d9462d2dbcead, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:14,078 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c0195071d4b74ea008bb625bd3316e78 in 455ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:14,083 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScanner/c0195071d4b74ea008bb625bd3316e78/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:14,084 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:37:14,084 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0195071d4b74ea008bb625bd3316e78: Waiting for close lock at 1733251033622Running coprocessor pre-close hooks at 1733251033622Disabling compacts and flushes for region at 1733251033622Disabling writes for close at 1733251033622Obtaining lock to block concurrent updates at 1733251033623 (+1 ms)Preparing flush snapshotting stores in c0195071d4b74ea008bb625bd3316e78 at 1733251033623Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251033623Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. at 1733251033624 (+1 ms)Flushing c0195071d4b74ea008bb625bd3316e78/testFamily: creating writer at 1733251033624Flushing c0195071d4b74ea008bb625bd3316e78/testFamily: appending metadata at 1733251033647 (+23 ms)Flushing c0195071d4b74ea008bb625bd3316e78/testFamily: closing flushed file at 1733251033647Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76450bbc: reopening flushed file at 1733251034067 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c0195071d4b74ea008bb625bd3316e78 in 455ms, sequenceid=8, compaction requested=false at 1733251034078 (+11 ms)Writing region close event to WAL at 1733251034079 (+1 ms)Running coprocessor post-close hooks at 1733251034084 (+5 ms)Closed at 1733251034084 2024-12-03T18:37:14,084 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733250974548.c0195071d4b74ea008bb625bd3316e78. 2024-12-03T18:37:14,085 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69c9b05ed64699ca6d63936f3e4cdfb2, disabling compactions & flushes 2024-12-03T18:37:14,085 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:37:14,085 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:37:14,085 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. after waiting 0 ms 2024-12-03T18:37:14,085 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:37:14,085 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 69c9b05ed64699ca6d63936f3e4cdfb2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:14,104 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2/.tmp/testFamily/0665c4ea28ab439cb7582d685a98fb5d is 61, key is row-0/testFamily:testQualifier/1733251015237/Put/seqid=0 2024-12-03T18:37:14,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741862_1038 (size=5049) 2024-12-03T18:37:14,117 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 69c9b05ed64699ca6d63936f3e4cdfb2, 6b667c859f6fac4fb5357ca15c20aa4e, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:14,118 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2/.tmp/testFamily/0665c4ea28ab439cb7582d685a98fb5d 2024-12-03T18:37:14,129 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2/.tmp/testFamily/0665c4ea28ab439cb7582d685a98fb5d as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2/testFamily/0665c4ea28ab439cb7582d685a98fb5d 2024-12-03T18:37:14,137 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2/testFamily/0665c4ea28ab439cb7582d685a98fb5d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:14,138 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 69c9b05ed64699ca6d63936f3e4cdfb2 in 53ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:14,143 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/69c9b05ed64699ca6d63936f3e4cdfb2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:14,143 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:37:14,144 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69c9b05ed64699ca6d63936f3e4cdfb2: Waiting for close lock at 1733251034085Running coprocessor pre-close hooks at 1733251034085Disabling compacts and flushes for region at 1733251034085Disabling writes for close at 1733251034085Obtaining lock to block concurrent updates at 1733251034085Preparing flush snapshotting stores in 69c9b05ed64699ca6d63936f3e4cdfb2 at 1733251034085Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251034085Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. at 1733251034086 (+1 ms)Flushing 69c9b05ed64699ca6d63936f3e4cdfb2/testFamily: creating writer at 1733251034086Flushing 69c9b05ed64699ca6d63936f3e4cdfb2/testFamily: appending metadata at 1733251034104 (+18 ms)Flushing 69c9b05ed64699ca6d63936f3e4cdfb2/testFamily: closing flushed file at 1733251034104Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2181a8f5: reopening flushed file at 1733251034128 (+24 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 69c9b05ed64699ca6d63936f3e4cdfb2 in 53ms, sequenceid=8, compaction requested=false at 1733251034138 (+10 ms)Writing region close event to WAL at 1733251034139 (+1 ms)Running coprocessor post-close hooks at 1733251034143 (+4 ms)Closed at 1733251034143 2024-12-03T18:37:14,144 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733251012187.69c9b05ed64699ca6d63936f3e4cdfb2. 2024-12-03T18:37:14,144 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6b667c859f6fac4fb5357ca15c20aa4e, disabling compactions & flushes 2024-12-03T18:37:14,144 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:37:14,144 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:37:14,144 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. after waiting 0 ms 2024-12-03T18:37:14,144 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:37:14,144 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6b667c859f6fac4fb5357ca15c20aa4e 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:14,159 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e/.tmp/testFamily/73f637df32414a789e65428d095c3511 is 61, key is row-0/testFamily:testQualifier/1733250995811/Put/seqid=0 2024-12-03T18:37:14,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741863_1039 (size=5049) 2024-12-03T18:37:14,318 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 6b667c859f6fac4fb5357ca15c20aa4e, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:14,518 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-12-03T18:37:14,518 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1325): Online Regions={6b667c859f6fac4fb5357ca15c20aa4e=testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e., eddb370b264893212cc95f0f0df239d4=hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4., 418b0b5898107dd4564a6a590a4f6738=testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738., 1a09aaf7de24e8ff87408593338defe9=hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9., f54ba82be14a86b5bcae0075aaf6d415=testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415.} 2024-12-03T18:37:14,518 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, 6b667c859f6fac4fb5357ca15c20aa4e, eddb370b264893212cc95f0f0df239d4, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:14,565 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e/.tmp/testFamily/73f637df32414a789e65428d095c3511 2024-12-03T18:37:14,579 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e/.tmp/testFamily/73f637df32414a789e65428d095c3511 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e/testFamily/73f637df32414a789e65428d095c3511 2024-12-03T18:37:14,592 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e/testFamily/73f637df32414a789e65428d095c3511, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:14,594 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6b667c859f6fac4fb5357ca15c20aa4e in 450ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:14,607 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextException/6b667c859f6fac4fb5357ca15c20aa4e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:14,609 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:37:14,609 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6b667c859f6fac4fb5357ca15c20aa4e: Waiting for close lock at 1733251034144Running coprocessor pre-close hooks at 1733251034144Disabling compacts and flushes for region at 1733251034144Disabling writes for close at 1733251034144Obtaining lock to block concurrent updates at 1733251034144Preparing flush snapshotting stores in 6b667c859f6fac4fb5357ca15c20aa4e at 1733251034144Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251034144Flushing stores of testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. at 1733251034145 (+1 ms)Flushing 6b667c859f6fac4fb5357ca15c20aa4e/testFamily: creating writer at 1733251034145Flushing 6b667c859f6fac4fb5357ca15c20aa4e/testFamily: appending metadata at 1733251034159 (+14 ms)Flushing 6b667c859f6fac4fb5357ca15c20aa4e/testFamily: closing flushed file at 1733251034159Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772e4ecc: reopening flushed file at 1733251034577 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6b667c859f6fac4fb5357ca15c20aa4e in 450ms, sequenceid=8, compaction requested=false at 1733251034594 (+17 ms)Writing region close event to WAL at 1733251034596 (+2 ms)Running coprocessor post-close hooks at 1733251034608 (+12 ms)Closed at 1733251034608 2024-12-03T18:37:14,609 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733250992766.6b667c859f6fac4fb5357ca15c20aa4e. 2024-12-03T18:37:14,609 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eddb370b264893212cc95f0f0df239d4, disabling compactions & flushes 2024-12-03T18:37:14,609 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:37:14,610 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:37:14,610 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. after waiting 0 ms 2024-12-03T18:37:14,610 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:37:14,610 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eddb370b264893212cc95f0f0df239d4 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:14,624 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T18:37:14,636 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4/.tmp/testFamily/1f3b0e8f03c945a5ab00074f62787376 is 61, key is row-0/testFamily:testQualifier/1733250986712/Put/seqid=0 2024-12-03T18:37:14,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741864_1040 (size=5049) 2024-12-03T18:37:14,655 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4/.tmp/testFamily/1f3b0e8f03c945a5ab00074f62787376 2024-12-03T18:37:14,667 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4/.tmp/testFamily/1f3b0e8f03c945a5ab00074f62787376 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4/testFamily/1f3b0e8f03c945a5ab00074f62787376 2024-12-03T18:37:14,680 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4/testFamily/1f3b0e8f03c945a5ab00074f62787376, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:14,683 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eddb370b264893212cc95f0f0df239d4 in 73ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:14,692 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNextAsync/eddb370b264893212cc95f0f0df239d4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:14,693 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:37:14,693 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eddb370b264893212cc95f0f0df239d4: Waiting for close lock at 1733251034609Running coprocessor pre-close hooks at 1733251034609Disabling compacts and flushes for region at 1733251034609Disabling writes for close at 1733251034610 (+1 ms)Obtaining lock to block concurrent updates at 1733251034610Preparing flush snapshotting stores in eddb370b264893212cc95f0f0df239d4 at 1733251034610Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251034610Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. at 1733251034611 (+1 ms)Flushing eddb370b264893212cc95f0f0df239d4/testFamily: creating writer at 1733251034611Flushing eddb370b264893212cc95f0f0df239d4/testFamily: appending metadata at 1733251034636 (+25 ms)Flushing eddb370b264893212cc95f0f0df239d4/testFamily: closing flushed file at 1733251034636Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39256543: reopening flushed file at 1733251034665 (+29 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eddb370b264893212cc95f0f0df239d4 in 73ms, sequenceid=8, compaction requested=false at 1733251034683 (+18 ms)Writing region close event to WAL at 1733251034687 (+4 ms)Running coprocessor post-close hooks at 1733251034693 (+6 ms)Closed at 1733251034693 2024-12-03T18:37:14,693 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733250983642.eddb370b264893212cc95f0f0df239d4. 2024-12-03T18:37:14,693 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 418b0b5898107dd4564a6a590a4f6738, disabling compactions & flushes 2024-12-03T18:37:14,693 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:37:14,694 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:37:14,694 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. after waiting 0 ms 2024-12-03T18:37:14,694 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:37:14,694 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 418b0b5898107dd4564a6a590a4f6738 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:14,717 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738/.tmp/testFamily/8d261601e6dc4f00a561805a2f692f74 is 61, key is row-0/testFamily:testQualifier/1733250974441/Put/seqid=0 2024-12-03T18:37:14,718 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:14,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741865_1041 (size=5049) 2024-12-03T18:37:14,919 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:15,119 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, 418b0b5898107dd4564a6a590a4f6738, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:15,123 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738/.tmp/testFamily/8d261601e6dc4f00a561805a2f692f74 2024-12-03T18:37:15,132 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738/.tmp/testFamily/8d261601e6dc4f00a561805a2f692f74 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738/testFamily/8d261601e6dc4f00a561805a2f692f74 2024-12-03T18:37:15,140 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738/testFamily/8d261601e6dc4f00a561805a2f692f74, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:15,142 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 418b0b5898107dd4564a6a590a4f6738 in 448ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:15,147 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testRetryOutOfOrderScannerNextExceptionAsync/418b0b5898107dd4564a6a590a4f6738/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:15,148 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:37:15,148 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 418b0b5898107dd4564a6a590a4f6738: Waiting for close lock at 1733251034693Running coprocessor pre-close hooks at 1733251034693Disabling compacts and flushes for region at 1733251034693Disabling writes for close at 1733251034694 (+1 ms)Obtaining lock to block concurrent updates at 1733251034694Preparing flush snapshotting stores in 418b0b5898107dd4564a6a590a4f6738 at 1733251034694Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251034694Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. at 1733251034695 (+1 ms)Flushing 418b0b5898107dd4564a6a590a4f6738/testFamily: creating writer at 1733251034695Flushing 418b0b5898107dd4564a6a590a4f6738/testFamily: appending metadata at 1733251034717 (+22 ms)Flushing 418b0b5898107dd4564a6a590a4f6738/testFamily: closing flushed file at 1733251034717Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ecde44f: reopening flushed file at 1733251035130 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 418b0b5898107dd4564a6a590a4f6738 in 448ms, sequenceid=8, compaction requested=false at 1733251035142 (+12 ms)Writing region close event to WAL at 1733251035143 (+1 ms)Running coprocessor post-close hooks at 1733251035148 (+5 ms)Closed at 1733251035148 2024-12-03T18:37:15,148 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733250971371.418b0b5898107dd4564a6a590a4f6738. 2024-12-03T18:37:15,149 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a09aaf7de24e8ff87408593338defe9, disabling compactions & flushes 2024-12-03T18:37:15,149 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:15,149 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:15,149 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. after waiting 0 ms 2024-12-03T18:37:15,149 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:15,149 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1a09aaf7de24e8ff87408593338defe9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:15,163 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9/.tmp/testFamily/36f05dd677644d13b1f940a2443582ba is 61, key is row-0/testFamily:testQualifier/1733251022318/Put/seqid=0 2024-12-03T18:37:15,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741866_1042 (size=5049) 2024-12-03T18:37:15,319 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:15,520 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T18:37:15,520 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1325): Online Regions={1a09aaf7de24e8ff87408593338defe9=hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9., f54ba82be14a86b5bcae0075aaf6d415=testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415.} 2024-12-03T18:37:15,520 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on 1a09aaf7de24e8ff87408593338defe9, f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:15,570 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9/.tmp/testFamily/36f05dd677644d13b1f940a2443582ba 2024-12-03T18:37:15,583 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9/.tmp/testFamily/36f05dd677644d13b1f940a2443582ba as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9/testFamily/36f05dd677644d13b1f940a2443582ba 2024-12-03T18:37:15,590 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9/testFamily/36f05dd677644d13b1f940a2443582ba, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:15,591 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1a09aaf7de24e8ff87408593338defe9 in 442ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:15,596 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/hbase/testMetaScanTimeoutOnNext/1a09aaf7de24e8ff87408593338defe9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:15,596 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:15,596 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a09aaf7de24e8ff87408593338defe9: Waiting for close lock at 1733251035148Running coprocessor pre-close hooks at 1733251035148Disabling compacts and flushes for region at 1733251035148Disabling writes for close at 1733251035149 (+1 ms)Obtaining lock to block concurrent updates at 1733251035149Preparing flush snapshotting stores in 1a09aaf7de24e8ff87408593338defe9 at 1733251035149Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251035149Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. at 1733251035150 (+1 ms)Flushing 1a09aaf7de24e8ff87408593338defe9/testFamily: creating writer at 1733251035150Flushing 1a09aaf7de24e8ff87408593338defe9/testFamily: appending metadata at 1733251035163 (+13 ms)Flushing 1a09aaf7de24e8ff87408593338defe9/testFamily: closing flushed file at 1733251035163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5589c4e3: reopening flushed file at 1733251035582 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1a09aaf7de24e8ff87408593338defe9 in 442ms, sequenceid=8, compaction requested=false at 1733251035591 (+9 ms)Writing region close event to WAL at 1733251035592 (+1 ms)Running coprocessor post-close hooks at 1733251035596 (+4 ms)Closed at 1733251035596 2024-12-03T18:37:15,597 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733251021290.1a09aaf7de24e8ff87408593338defe9. 2024-12-03T18:37:15,597 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f54ba82be14a86b5bcae0075aaf6d415, disabling compactions & flushes 2024-12-03T18:37:15,597 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:37:15,597 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:37:15,597 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. after waiting 0 ms 2024-12-03T18:37:15,597 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:37:15,597 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f54ba82be14a86b5bcae0075aaf6d415 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T18:37:15,612 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415/.tmp/testFamily/f387c5d3ac82495f996436f658014422 is 61, key is row-0/testFamily:testQualifier/1733251003011/Put/seqid=0 2024-12-03T18:37:15,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741867_1043 (size=5049) 2024-12-03T18:37:15,720 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:15,920 DEBUG [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1351): Waiting on f54ba82be14a86b5bcae0075aaf6d415 2024-12-03T18:37:16,017 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415/.tmp/testFamily/f387c5d3ac82495f996436f658014422 2024-12-03T18:37:16,027 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415/.tmp/testFamily/f387c5d3ac82495f996436f658014422 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415/testFamily/f387c5d3ac82495f996436f658014422 2024-12-03T18:37:16,036 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415/testFamily/f387c5d3ac82495f996436f658014422, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T18:37:16,037 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f54ba82be14a86b5bcae0075aaf6d415 in 440ms, sequenceid=8, compaction requested=false 2024-12-03T18:37:16,043 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/data/default/testNormalScanTimeoutOnNextAsync/f54ba82be14a86b5bcae0075aaf6d415/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T18:37:16,044 INFO [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:37:16,044 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f54ba82be14a86b5bcae0075aaf6d415: Waiting for close lock at 1733251035597Running coprocessor pre-close hooks at 1733251035597Disabling compacts and flushes for region at 1733251035597Disabling writes for close at 1733251035597Obtaining lock to block concurrent updates at 1733251035597Preparing flush snapshotting stores in f54ba82be14a86b5bcae0075aaf6d415 at 1733251035597Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733251035597Flushing stores of testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. at 1733251035598 (+1 ms)Flushing f54ba82be14a86b5bcae0075aaf6d415/testFamily: creating writer at 1733251035598Flushing f54ba82be14a86b5bcae0075aaf6d415/testFamily: appending metadata at 1733251035612 (+14 ms)Flushing f54ba82be14a86b5bcae0075aaf6d415/testFamily: closing flushed file at 1733251035612Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ed68014: reopening flushed file at 1733251036026 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f54ba82be14a86b5bcae0075aaf6d415 in 440ms, sequenceid=8, compaction requested=false at 1733251036037 (+11 ms)Writing region close event to WAL at 1733251036038 (+1 ms)Running coprocessor post-close hooks at 1733251036044 (+6 ms)Closed at 1733251036044 2024-12-03T18:37:16,044 DEBUG [RS_CLOSE_REGION-regionserver/5cd6afc4bcac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733251001978.f54ba82be14a86b5bcae0075aaf6d415. 2024-12-03T18:37:16,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T18:37:16,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T18:37:16,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T18:37:16,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T18:37:16,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T18:37:16,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T18:37:16,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T18:37:16,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T18:37:16,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T18:37:16,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T18:37:16,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T18:37:16,121 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(976): stopping server 5cd6afc4bcac,34371,1733250936949; all regions closed. 2024-12-03T18:37:16,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741834_1010 (size=19479) 2024-12-03T18:37:16,130 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949/5cd6afc4bcac%2C34371%2C1733250936949.meta.1733250970830.meta not finished, retry = 0 2024-12-03T18:37:16,238 DEBUG [RS:0;5cd6afc4bcac:34371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/oldWALs 2024-12-03T18:37:16,238 INFO [RS:0;5cd6afc4bcac:34371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5cd6afc4bcac%2C34371%2C1733250936949.meta:.meta(num 1733250970830) 2024-12-03T18:37:16,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741833_1009 (size=20031) 2024-12-03T18:37:16,241 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/WALs/5cd6afc4bcac,34371,1733250936949/5cd6afc4bcac%2C34371%2C1733250936949.1733250970292 not finished, retry = 0 2024-12-03T18:37:16,345 DEBUG [RS:0;5cd6afc4bcac:34371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/oldWALs 2024-12-03T18:37:16,345 INFO [RS:0;5cd6afc4bcac:34371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5cd6afc4bcac%2C34371%2C1733250936949:(num 1733250970292) 2024-12-03T18:37:16,345 DEBUG [RS:0;5cd6afc4bcac:34371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:37:16,345 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T18:37:16,345 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T18:37:16,346 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.ChoreService(370): Chore service for: regionserver/5cd6afc4bcac:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T18:37:16,346 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T18:37:16,346 INFO [regionserver/5cd6afc4bcac:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T18:37:16,347 INFO [RS:0;5cd6afc4bcac:34371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34371 2024-12-03T18:37:16,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T18:37:16,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5cd6afc4bcac,34371,1733250936949 2024-12-03T18:37:16,407 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T18:37:16,408 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5cd6afc4bcac,34371,1733250936949] 2024-12-03T18:37:16,423 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5cd6afc4bcac,34371,1733250936949 already deleted, retry=false 2024-12-03T18:37:16,423 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5cd6afc4bcac,34371,1733250936949 expired; onlineServers=0 2024-12-03T18:37:16,423 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5cd6afc4bcac,44487,1733250935430' ***** 2024-12-03T18:37:16,424 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T18:37:16,424 INFO [M:0;5cd6afc4bcac:44487 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T18:37:16,424 INFO [M:0;5cd6afc4bcac:44487 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T18:37:16,424 DEBUG [M:0;5cd6afc4bcac:44487 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T18:37:16,424 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T18:37:16,424 DEBUG [M:0;5cd6afc4bcac:44487 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T18:37:16,424 INFO [M:0;5cd6afc4bcac:44487 {}] hbase.ChoreService(370): Chore service for: master/5cd6afc4bcac:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T18:37:16,425 INFO [M:0;5cd6afc4bcac:44487 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T18:37:16,425 DEBUG [M:0;5cd6afc4bcac:44487 {}] master.HMaster(1795): Stopping service threads 2024-12-03T18:37:16,425 INFO [M:0;5cd6afc4bcac:44487 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T18:37:16,425 INFO [M:0;5cd6afc4bcac:44487 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T18:37:16,426 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster-HFileCleaner.small.0-1733250939999 {}] cleaner.HFileCleaner(306): Exit Thread[master/5cd6afc4bcac:0:becomeActiveMaster-HFileCleaner.small.0-1733250939999,5,FailOnTimeoutGroup] 2024-12-03T18:37:16,426 INFO [regionserver/5cd6afc4bcac:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T18:37:16,426 DEBUG [master/5cd6afc4bcac:0:becomeActiveMaster-HFileCleaner.large.0-1733250939996 {}] cleaner.HFileCleaner(306): Exit Thread[master/5cd6afc4bcac:0:becomeActiveMaster-HFileCleaner.large.0-1733250939996,5,FailOnTimeoutGroup] 2024-12-03T18:37:16,426 INFO [M:0;5cd6afc4bcac:44487 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T18:37:16,426 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T18:37:16,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T18:37:16,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:37:16,432 DEBUG [M:0;5cd6afc4bcac:44487 {}] zookeeper.ZKUtil(347): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T18:37:16,432 WARN [M:0;5cd6afc4bcac:44487 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T18:37:16,433 INFO [M:0;5cd6afc4bcac:44487 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/.lastflushedseqids 2024-12-03T18:37:16,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741868_1044 (size=712) 2024-12-03T18:37:16,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T18:37:16,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34371-0x1019dc3f80f0001, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T18:37:16,516 INFO [RS:0;5cd6afc4bcac:34371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T18:37:16,516 INFO [RS:0;5cd6afc4bcac:34371 {}] regionserver.HRegionServer(1031): Exiting; stopping=5cd6afc4bcac,34371,1733250936949; zookeeper connection closed. 2024-12-03T18:37:16,516 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@215b2a6f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@215b2a6f 2024-12-03T18:37:16,517 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T18:37:16,846 INFO [M:0;5cd6afc4bcac:44487 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T18:37:16,846 INFO [M:0;5cd6afc4bcac:44487 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T18:37:16,846 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T18:37:16,847 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:37:16,847 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:37:16,847 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T18:37:16,847 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:37:16,847 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.04 KB heapSize=191.06 KB 2024-12-03T18:37:16,871 DEBUG [M:0;5cd6afc4bcac:44487 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bae98c3d4b44cd493d75f75b944635d is 82, key is hbase:meta,,1/info:regioninfo/1733250970925/Put/seqid=0 2024-12-03T18:37:16,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741869_1045 (size=5672) 2024-12-03T18:37:16,886 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bae98c3d4b44cd493d75f75b944635d 2024-12-03T18:37:16,919 DEBUG [M:0;5cd6afc4bcac:44487 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/92ee9d86135f441bbe10286a5545931e is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733250972637/Put/seqid=0 2024-12-03T18:37:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741870_1046 (size=14067) 2024-12-03T18:37:16,929 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/92ee9d86135f441bbe10286a5545931e 2024-12-03T18:37:16,938 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92ee9d86135f441bbe10286a5545931e 2024-12-03T18:37:16,961 DEBUG [M:0;5cd6afc4bcac:44487 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/324218fccfd64d5998937d50d8d72ae8 is 69, key is 5cd6afc4bcac,34371,1733250936949/rs:state/1733250940089/Put/seqid=0 2024-12-03T18:37:16,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741871_1047 (size=5156) 2024-12-03T18:37:17,369 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/324218fccfd64d5998937d50d8d72ae8 2024-12-03T18:37:17,377 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bae98c3d4b44cd493d75f75b944635d as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bae98c3d4b44cd493d75f75b944635d 2024-12-03T18:37:17,386 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bae98c3d4b44cd493d75f75b944635d, entries=8, sequenceid=319, filesize=5.5 K 2024-12-03T18:37:17,388 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/92ee9d86135f441bbe10286a5545931e as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/92ee9d86135f441bbe10286a5545931e 2024-12-03T18:37:17,397 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92ee9d86135f441bbe10286a5545931e 2024-12-03T18:37:17,398 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/92ee9d86135f441bbe10286a5545931e, entries=33, sequenceid=319, filesize=13.7 K 2024-12-03T18:37:17,399 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/324218fccfd64d5998937d50d8d72ae8 as hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/324218fccfd64d5998937d50d8d72ae8 2024-12-03T18:37:17,412 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37123/user/jenkins/test-data/c031eaff-107e-e067-1438-63f3abba8061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/324218fccfd64d5998937d50d8d72ae8, entries=1, sequenceid=319, filesize=5.0 K 2024-12-03T18:37:17,415 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.04 KB/166955, heapSize ~190.77 KB/195344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 567ms, sequenceid=319, compaction requested=false 2024-12-03T18:37:17,427 INFO [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:37:17,427 DEBUG [M:0;5cd6afc4bcac:44487 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733251036846Disabling compacts and flushes for region at 1733251036846Disabling writes for close at 1733251036847 (+1 ms)Obtaining lock to block concurrent updates at 1733251036847Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733251036847Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166955, getHeapSize=195584, getOffHeapSize=0, getCellsCount=366 at 1733251036848 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733251036848Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733251036849 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733251036870 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733251036870Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733251036895 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733251036919 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733251036919Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733251036938 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733251036961 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733251036961Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@167642e0: reopening flushed file at 1733251037376 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e2ca511: reopening flushed file at 1733251037386 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d42d413: reopening flushed file at 1733251037398 (+12 ms)Finished flush of dataSize ~163.04 KB/166955, heapSize ~190.77 KB/195344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 567ms, sequenceid=319, compaction requested=false at 1733251037415 (+17 ms)Writing region close event to WAL at 1733251037426 (+11 ms)Closed at 1733251037426 2024-12-03T18:37:17,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36553 is added to blk_1073741830_1006 (size=188297) 2024-12-03T18:37:17,439 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T18:37:17,439 INFO [M:0;5cd6afc4bcac:44487 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T18:37:17,439 INFO [M:0;5cd6afc4bcac:44487 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44487 2024-12-03T18:37:17,439 INFO [M:0;5cd6afc4bcac:44487 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T18:37:17,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T18:37:17,565 INFO [M:0;5cd6afc4bcac:44487 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T18:37:17,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1019dc3f80f0000, quorum=127.0.0.1:53579, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T18:37:17,572 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T18:37:17,575 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T18:37:17,575 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T18:37:17,575 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T18:37:17,575 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/hadoop.log.dir/,STOPPED} 2024-12-03T18:37:17,580 WARN [BP-94149262-172.17.0.3-1733250930266 heartbeating to localhost/127.0.0.1:37123 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T18:37:17,580 WARN [BP-94149262-172.17.0.3-1733250930266 heartbeating to localhost/127.0.0.1:37123 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-94149262-172.17.0.3-1733250930266 (Datanode Uuid 82d58ec3-17d6-499f-86cb-8427e1f3e0fb) service to localhost/127.0.0.1:37123 2024-12-03T18:37:17,582 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722/data/data1/current/BP-94149262-172.17.0.3-1733250930266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T18:37:17,583 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T18:37:17,583 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T18:37:17,583 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/cluster_a16ad557-820f-757e-465f-8eb4b4cd4722/data/data2/current/BP-94149262-172.17.0.3-1733250930266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T18:37:17,584 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T18:37:17,595 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T18:37:17,599 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T18:37:17,599 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T18:37:17,599 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T18:37:17,599 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f63c2866-5669-f551-8756-02f4b8a37a89/hadoop.log.dir/,STOPPED} 2024-12-03T18:37:17,613 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T18:37:17,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down