2024-11-15 17:38:28,052 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-15 17:38:28,068 main DEBUG Took 0.015041 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 17:38:28,069 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 17:38:28,069 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 17:38:28,070 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 17:38:28,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,077 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 17:38:28,088 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,090 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,091 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,092 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,093 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,094 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,095 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,095 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,096 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,097 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,098 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 17:38:28,098 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,098 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 17:38:28,100 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 17:38:28,101 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 17:38:28,103 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 17:38:28,103 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 17:38:28,104 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 17:38:28,105 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 17:38:28,112 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 17:38:28,115 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 17:38:28,116 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 17:38:28,117 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 17:38:28,117 main DEBUG createAppenders(={Console}) 2024-11-15 17:38:28,118 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-15 17:38:28,118 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-15 17:38:28,118 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-15 17:38:28,118 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 17:38:28,119 main DEBUG OutputStream closed 2024-11-15 17:38:28,119 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 17:38:28,119 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 17:38:28,119 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-15 17:38:28,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 17:38:28,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 17:38:28,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 17:38:28,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 17:38:28,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 17:38:28,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 17:38:28,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 17:38:28,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 17:38:28,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 17:38:28,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 17:38:28,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 17:38:28,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 17:38:28,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 17:38:28,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 17:38:28,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 17:38:28,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 17:38:28,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 17:38:28,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 17:38:28,195 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 17:38:28,195 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@29ca3d04) with optional ClassLoader: null 2024-11-15 17:38:28,195 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 17:38:28,196 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@29ca3d04] started OK. 2024-11-15T17:38:28,207 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-15 17:38:28,210 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 17:38:28,210 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T17:38:28,431 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a 2024-11-15T17:38:28,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T17:38:28,473 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef, deleteOnExit=true 2024-11-15T17:38:28,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T17:38:28,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/test.cache.data in system properties and HBase conf 2024-11-15T17:38:28,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T17:38:28,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/hadoop.log.dir in system properties and HBase conf 2024-11-15T17:38:28,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T17:38:28,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T17:38:28,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T17:38:28,579 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T17:38:28,669 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T17:38:28,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T17:38:28,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T17:38:28,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T17:38:28,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T17:38:28,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T17:38:28,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T17:38:28,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T17:38:28,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T17:38:28,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T17:38:28,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/nfs.dump.dir in system properties and HBase conf 2024-11-15T17:38:28,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/java.io.tmpdir in system properties and HBase conf 2024-11-15T17:38:28,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T17:38:28,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T17:38:28,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T17:38:29,889 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T17:38:29,957 INFO [Time-limited test {}] log.Log(170): Logging initialized @2531ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T17:38:30,020 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T17:38:30,075 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T17:38:30,094 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T17:38:30,095 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T17:38:30,096 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T17:38:30,107 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T17:38:30,110 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@795e641e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/hadoop.log.dir/,AVAILABLE} 2024-11-15T17:38:30,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2183003e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T17:38:30,277 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@275a5b49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/java.io.tmpdir/jetty-localhost-38213-hadoop-hdfs-3_4_1-tests_jar-_-any-18188722614737309491/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T17:38:30,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6dc41719{HTTP/1.1, (http/1.1)}{localhost:38213} 2024-11-15T17:38:30,286 INFO [Time-limited test {}] server.Server(415): Started @2861ms 2024-11-15T17:38:30,895 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T17:38:30,902 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T17:38:30,903 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T17:38:30,903 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T17:38:30,904 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T17:38:30,905 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63308498{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/hadoop.log.dir/,AVAILABLE} 2024-11-15T17:38:30,905 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@262167d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T17:38:31,003 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@62138f9b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/java.io.tmpdir/jetty-localhost-45299-hadoop-hdfs-3_4_1-tests_jar-_-any-14803790801194668262/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T17:38:31,004 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4aadda0e{HTTP/1.1, (http/1.1)}{localhost:45299} 2024-11-15T17:38:31,004 INFO [Time-limited test {}] server.Server(415): Started @3579ms 2024-11-15T17:38:31,051 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T17:38:32,055 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef/data/data1/current/BP-132611115-172.17.0.2-1731692309276/current, will proceed with Du for space computation calculation, 2024-11-15T17:38:32,055 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef/data/data2/current/BP-132611115-172.17.0.2-1731692309276/current, will proceed with Du for space computation calculation, 2024-11-15T17:38:32,081 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T17:38:32,124 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaadd968d42d19853 with lease ID 0x9eb4c55f2924bf8f: Processing first storage report for DS-826df828-f103-4127-836d-aa21f28bfcb9 from datanode DatanodeRegistration(127.0.0.1:39187, datanodeUuid=5e4fbb00-1927-404a-863e-47921d0a9d1c, infoPort=46273, infoSecurePort=0, ipcPort=36705, storageInfo=lv=-57;cid=testClusterID;nsid=1063529548;c=1731692309276) 2024-11-15T17:38:32,125 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaadd968d42d19853 with lease ID 0x9eb4c55f2924bf8f: from storage DS-826df828-f103-4127-836d-aa21f28bfcb9 node DatanodeRegistration(127.0.0.1:39187, datanodeUuid=5e4fbb00-1927-404a-863e-47921d0a9d1c, infoPort=46273, infoSecurePort=0, ipcPort=36705, storageInfo=lv=-57;cid=testClusterID;nsid=1063529548;c=1731692309276), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T17:38:32,125 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaadd968d42d19853 with lease ID 0x9eb4c55f2924bf8f: Processing first storage report for DS-cfde9007-4113-4435-89d4-57c6544f360f from datanode DatanodeRegistration(127.0.0.1:39187, datanodeUuid=5e4fbb00-1927-404a-863e-47921d0a9d1c, infoPort=46273, infoSecurePort=0, ipcPort=36705, storageInfo=lv=-57;cid=testClusterID;nsid=1063529548;c=1731692309276) 2024-11-15T17:38:32,126 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaadd968d42d19853 with lease ID 0x9eb4c55f2924bf8f: from storage DS-cfde9007-4113-4435-89d4-57c6544f360f node DatanodeRegistration(127.0.0.1:39187, datanodeUuid=5e4fbb00-1927-404a-863e-47921d0a9d1c, infoPort=46273, infoSecurePort=0, ipcPort=36705, storageInfo=lv=-57;cid=testClusterID;nsid=1063529548;c=1731692309276), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T17:38:32,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a 2024-11-15T17:38:32,194 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef/zookeeper_0, clientPort=63121, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T17:38:32,203 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63121 2024-11-15T17:38:32,211 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:38:32,214 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:38:32,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741825_1001 (size=7) 2024-11-15T17:38:32,817 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 with version=8 2024-11-15T17:38:32,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/hbase-staging 2024-11-15T17:38:32,896 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T17:38:33,128 INFO [Time-limited test {}] client.ConnectionUtils(128): master/046d875b9f8f:0 server-side Connection retries=9 2024-11-15T17:38:33,137 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:38:33,137 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T17:38:33,142 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T17:38:33,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:38:33,142 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T17:38:33,262 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T17:38:33,312 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T17:38:33,320 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T17:38:33,324 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T17:38:33,345 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 130586 (auto-detected) 2024-11-15T17:38:33,346 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T17:38:33,363 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40087 2024-11-15T17:38:33,382 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40087 connecting to ZooKeeper ensemble=127.0.0.1:63121 2024-11-15T17:38:33,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400870x0, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T17:38:33,541 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40087-0x1013f93b21e0000 connected 2024-11-15T17:38:33,672 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:38:33,676 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:38:33,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:38:33,689 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30, hbase.cluster.distributed=false 2024-11-15T17:38:33,711 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T17:38:33,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-15T17:38:33,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40087 2024-11-15T17:38:33,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40087 2024-11-15T17:38:33,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-15T17:38:33,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40087 2024-11-15T17:38:33,817 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/046d875b9f8f:0 server-side Connection retries=9 2024-11-15T17:38:33,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:38:33,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T17:38:33,819 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T17:38:33,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T17:38:33,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T17:38:33,823 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T17:38:33,827 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T17:38:33,830 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40457 2024-11-15T17:38:33,833 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40457 connecting to ZooKeeper ensemble=127.0.0.1:63121 2024-11-15T17:38:33,835 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:38:33,839 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:38:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404570x0, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T17:38:33,916 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404570x0, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:38:33,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40457-0x1013f93b21e0001 connected 2024-11-15T17:38:33,919 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T17:38:33,927 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T17:38:33,930 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T17:38:33,935 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T17:38:33,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40457 2024-11-15T17:38:33,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40457 2024-11-15T17:38:33,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40457 2024-11-15T17:38:33,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40457 2024-11-15T17:38:33,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40457 2024-11-15T17:38:33,952 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;046d875b9f8f:40087 2024-11-15T17:38:33,954 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/046d875b9f8f,40087,1731692312977 2024-11-15T17:38:34,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:38:34,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:38:34,072 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/046d875b9f8f,40087,1731692312977 2024-11-15T17:38:34,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:34,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T17:38:34,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:34,102 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T17:38:34,103 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/046d875b9f8f,40087,1731692312977 from backup master directory 2024-11-15T17:38:34,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/046d875b9f8f,40087,1731692312977 2024-11-15T17:38:34,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:38:34,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T17:38:34,112 WARN [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T17:38:34,112 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=046d875b9f8f,40087,1731692312977 2024-11-15T17:38:34,114 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T17:38:34,116 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T17:38:34,167 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/hbase.id] with ID: ca110396-cccf-43e6-b595-bda8169d8a49 2024-11-15T17:38:34,167 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/.tmp/hbase.id 2024-11-15T17:38:34,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741826_1002 (size=42) 2024-11-15T17:38:34,580 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/.tmp/hbase.id]:[hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/hbase.id] 2024-11-15T17:38:34,632 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T17:38:34,638 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T17:38:34,656 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-15T17:38:34,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:34,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:34,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741827_1003 (size=196) 2024-11-15T17:38:35,154 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:38:35,157 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T17:38:35,171 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:38:35,174 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T17:38:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741828_1004 (size=1189) 2024-11-15T17:38:35,624 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store 2024-11-15T17:38:35,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741829_1005 (size=34) 2024-11-15T17:38:36,053 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T17:38:36,058 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:38:36,060 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T17:38:36,060 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:38:36,060 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:38:36,061 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T17:38:36,061 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:38:36,062 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:38:36,063 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731692316059Disabling compacts and flushes for region at 1731692316059Disabling writes for close at 1731692316061 (+2 ms)Writing region close event to WAL at 1731692316062 (+1 ms)Closed at 1731692316062 2024-11-15T17:38:36,065 WARN [master/046d875b9f8f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/.initializing 2024-11-15T17:38:36,065 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/WALs/046d875b9f8f,40087,1731692312977 2024-11-15T17:38:36,072 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T17:38:36,086 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=046d875b9f8f%2C40087%2C1731692312977, suffix=, logDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/WALs/046d875b9f8f,40087,1731692312977, archiveDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/oldWALs, maxLogs=10 2024-11-15T17:38:36,106 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/WALs/046d875b9f8f,40087,1731692312977/046d875b9f8f%2C40087%2C1731692312977.1731692316090, exclude list is [], retry=0 2024-11-15T17:38:36,121 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39187,DS-826df828-f103-4127-836d-aa21f28bfcb9,DISK] 2024-11-15T17:38:36,124 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T17:38:36,155 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/WALs/046d875b9f8f,40087,1731692312977/046d875b9f8f%2C40087%2C1731692312977.1731692316090 2024-11-15T17:38:36,156 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46273:46273)] 2024-11-15T17:38:36,157 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:38:36,157 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:38:36,160 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,161 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,198 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T17:38:36,228 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:36,230 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:38:36,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T17:38:36,235 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:36,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:38:36,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T17:38:36,240 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:36,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:38:36,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,244 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T17:38:36,245 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:36,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:38:36,246 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,250 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,251 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,256 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,257 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,261 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T17:38:36,265 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T17:38:36,270 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:38:36,271 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65669676, jitterRate=-0.02144557237625122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T17:38:36,281 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731692316173Initializing all the Stores at 1731692316176 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692316176Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692316177 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692316177Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692316177Cleaning up temporary data from old regions at 1731692316257 (+80 ms)Region opened successfully at 1731692316280 (+23 ms) 2024-11-15T17:38:36,282 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T17:38:36,320 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3643ea13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=046d875b9f8f/172.17.0.2:0 2024-11-15T17:38:36,350 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T17:38:36,360 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T17:38:36,360 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T17:38:36,363 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T17:38:36,364 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T17:38:36,369 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T17:38:36,369 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T17:38:36,393 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T17:38:36,401 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T17:38:36,448 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T17:38:36,451 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T17:38:36,453 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T17:38:36,467 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T17:38:36,470 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T17:38:36,477 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T17:38:36,490 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T17:38:36,491 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T17:38:36,500 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T17:38:36,522 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T17:38:36,532 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T17:38:36,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T17:38:36,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T17:38:36,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:36,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:36,548 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=046d875b9f8f,40087,1731692312977, sessionid=0x1013f93b21e0000, setting cluster-up flag (Was=false) 2024-11-15T17:38:36,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:36,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:36,606 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T17:38:36,609 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=046d875b9f8f,40087,1731692312977 2024-11-15T17:38:36,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:36,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:38:36,669 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T17:38:36,673 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=046d875b9f8f,40087,1731692312977 2024-11-15T17:38:36,682 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T17:38:36,744 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(746): ClusterId : ca110396-cccf-43e6-b595-bda8169d8a49 2024-11-15T17:38:36,746 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T17:38:36,747 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T17:38:36,755 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T17:38:36,755 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T17:38:36,756 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T17:38:36,762 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T17:38:36,765 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T17:38:36,765 DEBUG [RS:0;046d875b9f8f:40457 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51524ac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=046d875b9f8f/172.17.0.2:0 2024-11-15T17:38:36,769 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 046d875b9f8f,40087,1731692312977 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T17:38:36,777 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/046d875b9f8f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:38:36,777 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/046d875b9f8f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:38:36,778 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/046d875b9f8f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:38:36,778 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/046d875b9f8f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T17:38:36,778 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/046d875b9f8f:0, corePoolSize=10, maxPoolSize=10 2024-11-15T17:38:36,778 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,778 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/046d875b9f8f:0, corePoolSize=2, maxPoolSize=2 2024-11-15T17:38:36,778 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,779 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;046d875b9f8f:40457 2024-11-15T17:38:36,780 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731692346780 2024-11-15T17:38:36,781 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T17:38:36,782 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T17:38:36,782 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T17:38:36,782 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T17:38:36,782 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T17:38:36,784 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T17:38:36,784 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(2659): reportForDuty to master=046d875b9f8f,40087,1731692312977 with port=40457, startcode=1731692313785 2024-11-15T17:38:36,785 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T17:38:36,786 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T17:38:36,786 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T17:38:36,786 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T17:38:36,786 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T17:38:36,787 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,791 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:36,791 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T17:38:36,791 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T17:38:36,792 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T17:38:36,793 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T17:38:36,795 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T17:38:36,795 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T17:38:36,796 DEBUG [RS:0;046d875b9f8f:40457 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T17:38:36,797 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/046d875b9f8f:0:becomeActiveMaster-HFileCleaner.large.0-1731692316797,5,FailOnTimeoutGroup] 2024-11-15T17:38:36,799 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/046d875b9f8f:0:becomeActiveMaster-HFileCleaner.small.0-1731692316798,5,FailOnTimeoutGroup] 2024-11-15T17:38:36,799 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,799 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T17:38:36,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741831_1007 (size=1321) 2024-11-15T17:38:36,801 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,801 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56005, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T17:38:36,831 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 046d875b9f8f,40457,1731692313785 2024-11-15T17:38:36,833 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40087 {}] master.ServerManager(517): Registering regionserver=046d875b9f8f,40457,1731692313785 2024-11-15T17:38:36,847 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:38:36,847 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45921 2024-11-15T17:38:36,847 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T17:38:36,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T17:38:36,859 DEBUG [RS:0;046d875b9f8f:40457 {}] zookeeper.ZKUtil(111): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/046d875b9f8f,40457,1731692313785 2024-11-15T17:38:36,859 WARN [RS:0;046d875b9f8f:40457 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T17:38:36,860 INFO [RS:0;046d875b9f8f:40457 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T17:38:36,860 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785 2024-11-15T17:38:36,862 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [046d875b9f8f,40457,1731692313785] 2024-11-15T17:38:36,886 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T17:38:36,903 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T17:38:36,907 INFO [RS:0;046d875b9f8f:40457 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T17:38:36,907 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,908 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T17:38:36,913 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T17:38:36,914 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,915 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,915 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,915 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,915 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,915 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,915 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/046d875b9f8f:0, corePoolSize=2, maxPoolSize=2 2024-11-15T17:38:36,915 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,916 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,916 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,916 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,916 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,916 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/046d875b9f8f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T17:38:36,916 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/046d875b9f8f:0, corePoolSize=3, maxPoolSize=3 2024-11-15T17:38:36,917 DEBUG [RS:0;046d875b9f8f:40457 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/046d875b9f8f:0, corePoolSize=3, maxPoolSize=3 2024-11-15T17:38:36,917 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,918 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,918 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,918 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,918 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,918 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40457,1731692313785-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T17:38:36,937 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T17:38:36,938 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40457,1731692313785-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,938 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,939 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.Replication(171): 046d875b9f8f,40457,1731692313785 started 2024-11-15T17:38:36,956 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:38:36,956 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1482): Serving as 046d875b9f8f,40457,1731692313785, RpcServer on 046d875b9f8f/172.17.0.2:40457, sessionid=0x1013f93b21e0001 2024-11-15T17:38:36,957 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T17:38:36,957 DEBUG [RS:0;046d875b9f8f:40457 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 046d875b9f8f,40457,1731692313785 2024-11-15T17:38:36,957 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '046d875b9f8f,40457,1731692313785' 2024-11-15T17:38:36,957 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T17:38:36,959 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T17:38:36,960 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T17:38:36,960 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T17:38:36,960 DEBUG [RS:0;046d875b9f8f:40457 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 046d875b9f8f,40457,1731692313785 2024-11-15T17:38:36,960 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '046d875b9f8f,40457,1731692313785' 2024-11-15T17:38:36,960 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T17:38:36,961 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T17:38:36,961 DEBUG [RS:0;046d875b9f8f:40457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T17:38:36,961 INFO [RS:0;046d875b9f8f:40457 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T17:38:36,962 INFO [RS:0;046d875b9f8f:40457 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T17:38:37,208 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T17:38:37,210 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:38:37,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741832_1008 (size=32) 2024-11-15T17:38:37,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:38:37,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T17:38:37,636 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T17:38:37,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:37,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:38:37,637 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T17:38:37,640 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T17:38:37,640 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:37,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:38:37,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T17:38:37,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T17:38:37,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:37,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:38:37,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T17:38:37,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T17:38:37,647 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:38:37,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:38:37,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T17:38:37,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740 2024-11-15T17:38:37,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740 2024-11-15T17:38:37,653 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T17:38:37,653 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T17:38:37,654 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T17:38:37,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T17:38:37,659 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:38:37,660 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72995258, jitterRate=0.08771410584449768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T17:38:37,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731692317628Initializing all the Stores at 1731692317632 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692317632Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692317632Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692317633 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692317633Cleaning up temporary data from old regions at 1731692317653 (+20 ms)Region opened successfully at 1731692317663 (+10 ms) 2024-11-15T17:38:37,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T17:38:37,663 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T17:38:37,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T17:38:37,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T17:38:37,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T17:38:37,665 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T17:38:37,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731692317663Disabling compacts and flushes for region at 1731692317663Disabling writes for close at 1731692317663Writing region close event to WAL at 1731692317664 (+1 ms)Closed at 1731692317665 (+1 ms) 2024-11-15T17:38:37,668 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T17:38:37,668 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T17:38:37,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T17:38:37,680 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T17:38:37,683 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T17:38:37,835 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:38,837 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:39,838 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:40,839 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:41,841 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:42,842 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:43,311 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T17:38:43,312 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T17:38:43,843 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:44,845 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:45,846 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:46,847 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:47,848 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:48,849 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:49,851 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:50,852 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:51,853 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:52,854 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:53,856 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:54,857 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:55,858 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:56,859 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:57,860 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:58,861 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:38:59,862 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:00,863 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:01,864 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:02,127 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:39:02,866 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:03,867 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:04,868 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:05,869 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:06,870 WARN [046d875b9f8f:40087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T17:39:06,974 INFO [RS:0;046d875b9f8f:40457 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T17:39:06,978 INFO [RS:0;046d875b9f8f:40457 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=046d875b9f8f%2C40457%2C1731692313785, suffix=, logDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785, archiveDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/oldWALs, maxLogs=32 2024-11-15T17:39:06,993 DEBUG [RS:0;046d875b9f8f:40457 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785/046d875b9f8f%2C40457%2C1731692313785.1731692346980, exclude list is [], retry=0 2024-11-15T17:39:06,998 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39187,DS-826df828-f103-4127-836d-aa21f28bfcb9,DISK] 2024-11-15T17:39:07,001 INFO [RS:0;046d875b9f8f:40457 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785/046d875b9f8f%2C40457%2C1731692313785.1731692346980 2024-11-15T17:39:07,002 DEBUG [RS:0;046d875b9f8f:40457 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46273:46273)] 2024-11-15T17:39:07,126 DEBUG [046d875b9f8f:40087 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T17:39:07,142 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:07,147 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 046d875b9f8f,40457,1731692313785, state=OPENING 2024-11-15T17:39:07,202 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T17:39:07,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:07,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:39:07,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:07,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:07,218 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T17:39:07,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:07,403 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T17:39:07,406 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41467, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T17:39:07,417 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T17:39:07,417 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T17:39:07,418 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T17:39:07,421 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=046d875b9f8f%2C40457%2C1731692313785.meta, suffix=.meta, logDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785, archiveDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/oldWALs, maxLogs=32 2024-11-15T17:39:07,442 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785/046d875b9f8f%2C40457%2C1731692313785.meta.1731692347424.meta, exclude list is [], retry=0 2024-11-15T17:39:07,446 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39187,DS-826df828-f103-4127-836d-aa21f28bfcb9,DISK] 2024-11-15T17:39:07,449 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785/046d875b9f8f%2C40457%2C1731692313785.meta.1731692347424.meta 2024-11-15T17:39:07,449 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46273:46273)] 2024-11-15T17:39:07,450 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:07,451 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T17:39:07,454 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T17:39:07,458 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T17:39:07,461 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T17:39:07,462 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:07,462 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T17:39:07,462 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T17:39:07,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T17:39:07,466 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T17:39:07,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:07,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:07,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T17:39:07,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T17:39:07,469 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:07,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:07,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T17:39:07,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T17:39:07,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:07,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:07,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T17:39:07,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T17:39:07,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:07,475 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T17:39:07,476 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T17:39:07,477 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740 2024-11-15T17:39:07,480 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740 2024-11-15T17:39:07,483 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T17:39:07,483 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T17:39:07,484 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T17:39:07,486 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T17:39:07,488 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63966130, jitterRate=-0.04683038592338562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T17:39:07,488 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T17:39:07,490 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731692347462Writing region info on filesystem at 1731692347463 (+1 ms)Initializing all the Stores at 1731692347464 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692347464Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692347465 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692347465Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731692347465Cleaning up temporary data from old regions at 1731692347483 (+18 ms)Running coprocessor post-open hooks at 1731692347488 (+5 ms)Region opened successfully at 1731692347489 (+1 ms) 2024-11-15T17:39:07,496 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731692347396 2024-11-15T17:39:07,506 DEBUG [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T17:39:07,507 INFO [RS_OPEN_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T17:39:07,509 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:07,512 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 046d875b9f8f,40457,1731692313785, state=OPEN 2024-11-15T17:39:07,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T17:39:07,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T17:39:07,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:07,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T17:39:07,549 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:07,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T17:39:07,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=046d875b9f8f,40457,1731692313785 in 328 msec 2024-11-15T17:39:07,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T17:39:07,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8840 sec 2024-11-15T17:39:07,564 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T17:39:07,565 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T17:39:07,582 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:07,583 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=046d875b9f8f,40457,1731692313785, seqNum=-1] 2024-11-15T17:39:07,602 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:07,605 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53559, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:07,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9140 sec 2024-11-15T17:39:07,625 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731692347625, completionTime=-1 2024-11-15T17:39:07,628 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T17:39:07,628 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T17:39:07,651 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-15T17:39:07,651 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731692407651 2024-11-15T17:39:07,651 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731692467651 2024-11-15T17:39:07,651 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-15T17:39:07,653 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40087,1731692312977-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:07,653 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40087,1731692312977-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:07,653 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40087,1731692312977-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:07,654 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-046d875b9f8f:40087, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:07,654 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:07,655 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:07,660 DEBUG [master/046d875b9f8f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T17:39:07,680 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.567sec 2024-11-15T17:39:07,681 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T17:39:07,682 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T17:39:07,683 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T17:39:07,684 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T17:39:07,684 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T17:39:07,685 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40087,1731692312977-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T17:39:07,685 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40087,1731692312977-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T17:39:07,692 DEBUG [master/046d875b9f8f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T17:39:07,693 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T17:39:07,693 INFO [master/046d875b9f8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=046d875b9f8f,40087,1731692312977-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T17:39:07,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53deffd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:07,695 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T17:39:07,695 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T17:39:07,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 046d875b9f8f,40087,-1 for getting cluster id 2024-11-15T17:39:07,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:07,707 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ca110396-cccf-43e6-b595-bda8169d8a49' 2024-11-15T17:39:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:07,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ca110396-cccf-43e6-b595-bda8169d8a49" 2024-11-15T17:39:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18214de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [046d875b9f8f,40087,-1] 2024-11-15T17:39:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:07,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:07,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:07,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26988b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:07,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=046d875b9f8f,40457,1731692313785, seqNum=-1] 2024-11-15T17:39:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:07,730 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:07,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=046d875b9f8f,40087,1731692312977 2024-11-15T17:39:07,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627c0060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:07,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 046d875b9f8f,40087,-1 for getting cluster id 2024-11-15T17:39:07,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T17:39:07,757 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ca110396-cccf-43e6-b595-bda8169d8a49' 2024-11-15T17:39:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T17:39:07,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ca110396-cccf-43e6-b595-bda8169d8a49" 2024-11-15T17:39:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2adcb0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [046d875b9f8f,40087,-1] 2024-11-15T17:39:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T17:39:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:39:07,760 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T17:39:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a188c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T17:39:07,776 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=1017 2024-11-15T17:39:07,783 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T17:39:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 046d875b9f8f,40087,1731692312977 2024-11-15T17:39:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@494ab692 2024-11-15T17:39:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T17:39:07,810 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T17:39:07,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-15T17:39:07,826 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:07,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-15T17:39:07,829 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:07,831 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:07,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741835_1011 (size=428) 2024-11-15T17:39:08,250 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e60caf819870f5aeb7177660cf6de15a, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:08,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741836_1012 (size=79) 2024-11-15T17:39:08,262 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:08,262 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing e60caf819870f5aeb7177660cf6de15a, disabling compactions & flushes 2024-11-15T17:39:08,262 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:39:08,262 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:39:08,262 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. after waiting 0 ms 2024-11-15T17:39:08,262 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:39:08,262 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:39:08,263 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e60caf819870f5aeb7177660cf6de15a: Waiting for close lock at 1731692348262Disabling compacts and flushes for region at 1731692348262Disabling writes for close at 1731692348262Writing region close event to WAL at 1731692348262Closed at 1731692348262 2024-11-15T17:39:08,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:08,271 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731692348266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692348266"}]},"ts":"1731692348266"} 2024-11-15T17:39:08,277 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:08,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:08,283 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692348279"}]},"ts":"1731692348279"} 2024-11-15T17:39:08,288 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-15T17:39:08,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e60caf819870f5aeb7177660cf6de15a, ASSIGN}] 2024-11-15T17:39:08,294 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e60caf819870f5aeb7177660cf6de15a, ASSIGN 2024-11-15T17:39:08,296 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e60caf819870f5aeb7177660cf6de15a, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:08,450 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e60caf819870f5aeb7177660cf6de15a, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:08,459 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e60caf819870f5aeb7177660cf6de15a, ASSIGN because future has completed 2024-11-15T17:39:08,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e60caf819870f5aeb7177660cf6de15a, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:08,629 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:39:08,630 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e60caf819870f5aeb7177660cf6de15a, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:08,631 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,631 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:08,631 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,632 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,635 INFO [StoreOpener-e60caf819870f5aeb7177660cf6de15a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,638 INFO [StoreOpener-e60caf819870f5aeb7177660cf6de15a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e60caf819870f5aeb7177660cf6de15a columnFamilyName testFamily 2024-11-15T17:39:08,638 DEBUG [StoreOpener-e60caf819870f5aeb7177660cf6de15a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:08,639 INFO [StoreOpener-e60caf819870f5aeb7177660cf6de15a-1 {}] regionserver.HStore(327): Store=e60caf819870f5aeb7177660cf6de15a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:08,640 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,641 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,642 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,642 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,643 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,646 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,649 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:08,650 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e60caf819870f5aeb7177660cf6de15a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59299192, jitterRate=-0.11637318134307861}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:08,650 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:39:08,652 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e60caf819870f5aeb7177660cf6de15a: Running coprocessor pre-open hook at 1731692348632Writing region info on filesystem at 1731692348632Initializing all the Stores at 1731692348634 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692348635 (+1 ms)Cleaning up temporary data from old regions at 1731692348643 (+8 ms)Running coprocessor post-open hooks at 1731692348650 (+7 ms)Region opened successfully at 1731692348652 (+2 ms) 2024-11-15T17:39:08,654 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a., pid=6, masterSystemTime=1731692348615 2024-11-15T17:39:08,657 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:39:08,657 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:39:08,658 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e60caf819870f5aeb7177660cf6de15a, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:08,662 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e60caf819870f5aeb7177660cf6de15a, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:08,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T17:39:08,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e60caf819870f5aeb7177660cf6de15a, server=046d875b9f8f,40457,1731692313785 in 204 msec 2024-11-15T17:39:08,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T17:39:08,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e60caf819870f5aeb7177660cf6de15a, ASSIGN in 378 msec 2024-11-15T17:39:08,674 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:08,674 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692348674"}]},"ts":"1731692348674"} 2024-11-15T17:39:08,677 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-15T17:39:08,679 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:08,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 861 msec 2024-11-15T17:39:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T17:39:08,859 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-15T17:39:08,859 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-15T17:39:08,861 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:08,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-15T17:39:08,868 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:08,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-15T17:39:08,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T17:39:08,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=046d875b9f8f,40457,1731692313785, seqNum=-1] 2024-11-15T17:39:08,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T17:39:08,875 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T17:39:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:08,894 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:08,898 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-15T17:39:08,902 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-15T17:39:08,907 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-15T17:39:08,909 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-15T17:39:08,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6272187458155184120 2024-11-15T17:39:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692351908, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:08,933 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-15T17:39:08,935 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-15T17:39:08,936 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:08,937 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:08,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6272187458155184118 2024-11-15T17:39:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692351937, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:08,942 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:08,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6272187458155184117 2024-11-15T17:39:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692351942, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:08,947 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:08,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6272187458155184116 2024-11-15T17:39:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692351947, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:08,951 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:08,962 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-495086358_22 at /127.0.0.1:54936 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=165 (was 165), ProcessCount=11 (was 11), AvailableMemoryMB=972 (was 1017) 2024-11-15T17:39:08,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=971 2024-11-15T17:39:08,975 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:08,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-15T17:39:08,979 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:08,979 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:08,979 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-15T17:39:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:08,981 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:08,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741837_1013 (size=414) 2024-11-15T17:39:09,399 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b6cd4b9f27db5117c18715b0b3667bc7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:09,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741838_1014 (size=65) 2024-11-15T17:39:09,810 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:09,810 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing b6cd4b9f27db5117c18715b0b3667bc7, disabling compactions & flushes 2024-11-15T17:39:09,810 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:39:09,810 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:39:09,811 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. after waiting 0 ms 2024-11-15T17:39:09,811 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:39:09,811 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:39:09,811 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for b6cd4b9f27db5117c18715b0b3667bc7: Waiting for close lock at 1731692349810Disabling compacts and flushes for region at 1731692349810Disabling writes for close at 1731692349811 (+1 ms)Writing region close event to WAL at 1731692349811Closed at 1731692349811 2024-11-15T17:39:09,816 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:09,816 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731692349816"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692349816"}]},"ts":"1731692349816"} 2024-11-15T17:39:09,821 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:09,823 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:09,823 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692349823"}]},"ts":"1731692349823"} 2024-11-15T17:39:09,826 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-15T17:39:09,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b6cd4b9f27db5117c18715b0b3667bc7, ASSIGN}] 2024-11-15T17:39:09,828 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b6cd4b9f27db5117c18715b0b3667bc7, ASSIGN 2024-11-15T17:39:09,830 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b6cd4b9f27db5117c18715b0b3667bc7, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:09,981 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b6cd4b9f27db5117c18715b0b3667bc7, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:09,986 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b6cd4b9f27db5117c18715b0b3667bc7, ASSIGN because future has completed 2024-11-15T17:39:09,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6cd4b9f27db5117c18715b0b3667bc7, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:10,154 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:39:10,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b6cd4b9f27db5117c18715b0b3667bc7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:10,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:10,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,158 INFO [StoreOpener-b6cd4b9f27db5117c18715b0b3667bc7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,161 INFO [StoreOpener-b6cd4b9f27db5117c18715b0b3667bc7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6cd4b9f27db5117c18715b0b3667bc7 columnFamilyName testFamily 2024-11-15T17:39:10,161 DEBUG [StoreOpener-b6cd4b9f27db5117c18715b0b3667bc7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:10,162 INFO [StoreOpener-b6cd4b9f27db5117c18715b0b3667bc7-1 {}] regionserver.HStore(327): Store=b6cd4b9f27db5117c18715b0b3667bc7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:10,162 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,164 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,165 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,166 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,166 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,171 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,175 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:10,176 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened b6cd4b9f27db5117c18715b0b3667bc7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63166708, jitterRate=-0.0587427020072937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:10,176 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:39:10,177 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b6cd4b9f27db5117c18715b0b3667bc7: Running coprocessor pre-open hook at 1731692350156Writing region info on filesystem at 1731692350156Initializing all the Stores at 1731692350157 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692350158 (+1 ms)Cleaning up temporary data from old regions at 1731692350166 (+8 ms)Running coprocessor post-open hooks at 1731692350176 (+10 ms)Region opened successfully at 1731692350177 (+1 ms) 2024-11-15T17:39:10,179 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., pid=9, masterSystemTime=1731692350143 2024-11-15T17:39:10,183 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:39:10,183 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:39:10,184 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b6cd4b9f27db5117c18715b0b3667bc7, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:10,188 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6cd4b9f27db5117c18715b0b3667bc7, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:10,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-15T17:39:10,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure b6cd4b9f27db5117c18715b0b3667bc7, server=046d875b9f8f,40457,1731692313785 in 203 msec 2024-11-15T17:39:10,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T17:39:10,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=b6cd4b9f27db5117c18715b0b3667bc7, ASSIGN in 368 msec 2024-11-15T17:39:10,199 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:10,199 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692350199"}]},"ts":"1731692350199"} 2024-11-15T17:39:10,202 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-15T17:39:10,203 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:10,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2290 sec 2024-11-15T17:39:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T17:39:12,028 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-15T17:39:12,029 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-15T17:39:12,029 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:12,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-15T17:39:12,038 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:12,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-15T17:39:12,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:12,055 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:12,055 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-15T17:39:12,058 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-15T17:39:12,060 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-15T17:39:12,997 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:39:13,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-15T17:39:13,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T17:39:13,032 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-15T17:39:13,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T17:39:13,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:13,313 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T17:39:13,314 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T17:39:13,315 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-15T17:39:13,315 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:13,317 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-15T17:39:13,317 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:18,064 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., hostname=046d875b9f8f,40457,1731692313785, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=046d875b9f8f:40457 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-15T17:39:18,065 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-15T17:39:18,065 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 from cache 2024-11-15T17:39:18,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 165), ProcessCount=11 (was 11), AvailableMemoryMB=934 (was 971) 2024-11-15T17:39:18,083 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=933 2024-11-15T17:39:18,086 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:18,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-15T17:39:18,090 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:18,090 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:18,090 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-15T17:39:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-15T17:39:18,092 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:18,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741839_1015 (size=412) 2024-11-15T17:39:18,538 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 95717f6f1db1677b5c9dfc55756121bc, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:18,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741840_1016 (size=63) 2024-11-15T17:39:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-15T17:39:18,955 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:18,955 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 95717f6f1db1677b5c9dfc55756121bc, disabling compactions & flushes 2024-11-15T17:39:18,955 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:39:18,955 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:39:18,955 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. after waiting 0 ms 2024-11-15T17:39:18,955 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:39:18,956 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:39:18,956 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 95717f6f1db1677b5c9dfc55756121bc: Waiting for close lock at 1731692358955Disabling compacts and flushes for region at 1731692358955Disabling writes for close at 1731692358955Writing region close event to WAL at 1731692358956 (+1 ms)Closed at 1731692358956 2024-11-15T17:39:18,958 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:18,958 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731692358958"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692358958"}]},"ts":"1731692358958"} 2024-11-15T17:39:18,961 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:18,963 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:18,963 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692358963"}]},"ts":"1731692358963"} 2024-11-15T17:39:18,966 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-15T17:39:18,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=95717f6f1db1677b5c9dfc55756121bc, ASSIGN}] 2024-11-15T17:39:18,969 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=95717f6f1db1677b5c9dfc55756121bc, ASSIGN 2024-11-15T17:39:18,970 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=95717f6f1db1677b5c9dfc55756121bc, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-15T17:39:19,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=95717f6f1db1677b5c9dfc55756121bc, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:19,125 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=95717f6f1db1677b5c9dfc55756121bc, ASSIGN because future has completed 2024-11-15T17:39:19,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95717f6f1db1677b5c9dfc55756121bc, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:19,286 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:39:19,286 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 95717f6f1db1677b5c9dfc55756121bc, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:19,286 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,287 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:19,287 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,287 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,289 INFO [StoreOpener-95717f6f1db1677b5c9dfc55756121bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,291 INFO [StoreOpener-95717f6f1db1677b5c9dfc55756121bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95717f6f1db1677b5c9dfc55756121bc columnFamilyName testFamily 2024-11-15T17:39:19,291 DEBUG [StoreOpener-95717f6f1db1677b5c9dfc55756121bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:19,292 INFO [StoreOpener-95717f6f1db1677b5c9dfc55756121bc-1 {}] regionserver.HStore(327): Store=95717f6f1db1677b5c9dfc55756121bc/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:19,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,294 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,297 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:19,301 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 95717f6f1db1677b5c9dfc55756121bc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68702934, jitterRate=0.023753494024276733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:19,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:39:19,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 95717f6f1db1677b5c9dfc55756121bc: Running coprocessor pre-open hook at 1731692359287Writing region info on filesystem at 1731692359287Initializing all the Stores at 1731692359288 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692359288Cleaning up temporary data from old regions at 1731692359294 (+6 ms)Running coprocessor post-open hooks at 1731692359301 (+7 ms)Region opened successfully at 1731692359302 (+1 ms) 2024-11-15T17:39:19,303 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc., pid=12, masterSystemTime=1731692359280 2024-11-15T17:39:19,306 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:39:19,307 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:39:19,307 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=95717f6f1db1677b5c9dfc55756121bc, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:19,310 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95717f6f1db1677b5c9dfc55756121bc, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:19,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T17:39:19,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 95717f6f1db1677b5c9dfc55756121bc, server=046d875b9f8f,40457,1731692313785 in 186 msec 2024-11-15T17:39:19,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-15T17:39:19,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=95717f6f1db1677b5c9dfc55756121bc, ASSIGN in 350 msec 2024-11-15T17:39:19,320 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:19,320 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692359320"}]},"ts":"1731692359320"} 2024-11-15T17:39:19,323 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-15T17:39:19,324 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:19,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2380 sec 2024-11-15T17:39:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-15T17:39:21,129 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-15T17:39:21,129 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-15T17:39:21,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:21,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-15T17:39:21,138 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:21,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-15T17:39:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:21,157 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:21,157 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-15T17:39:21,160 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-15T17:39:21,163 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-15T17:39:21,164 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-15T17:39:21,165 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-15T17:39:23,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-15T17:39:23,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:24,822 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:39:25,034 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-15T17:39:27,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184107' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-15T17:39:27,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:37102 deadline: 1731692373172, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184107' does not exist 2024-11-15T17:39:27,177 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 046d875b9f8f,40457,1731692313785 for closing scanner id = -6272187458155184107 for 95717f6f1db1677b5c9dfc55756121bc of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184107' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:27,180 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=149 (was 140) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2071 (was 933) - AvailableMemoryMB LEAK? - 2024-11-15T17:39:27,190 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=149, ProcessCount=11, AvailableMemoryMB=2071 2024-11-15T17:39:27,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-15T17:39:27,199 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:27,199 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:27,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-15T17:39:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:27,202 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:27,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741841_1017 (size=423) 2024-11-15T17:39:27,619 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8d71afd5b7fbbee97ab8477168698886, NAME => 'testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:27,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741842_1018 (size=74) 2024-11-15T17:39:27,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-15T17:39:28,032 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:28,032 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 8d71afd5b7fbbee97ab8477168698886, disabling compactions & flushes 2024-11-15T17:39:28,032 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:39:28,033 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:39:28,033 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. after waiting 0 ms 2024-11-15T17:39:28,033 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:39:28,033 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:39:28,033 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8d71afd5b7fbbee97ab8477168698886: Waiting for close lock at 1731692368032Disabling compacts and flushes for region at 1731692368032Disabling writes for close at 1731692368033 (+1 ms)Writing region close event to WAL at 1731692368033Closed at 1731692368033 2024-11-15T17:39:28,035 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:28,035 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731692368035"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692368035"}]},"ts":"1731692368035"} 2024-11-15T17:39:28,039 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:28,040 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:28,041 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692368040"}]},"ts":"1731692368040"} 2024-11-15T17:39:28,044 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-15T17:39:28,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8d71afd5b7fbbee97ab8477168698886, ASSIGN}] 2024-11-15T17:39:28,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8d71afd5b7fbbee97ab8477168698886, ASSIGN 2024-11-15T17:39:28,047 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8d71afd5b7fbbee97ab8477168698886, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:28,198 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8d71afd5b7fbbee97ab8477168698886, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:28,202 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8d71afd5b7fbbee97ab8477168698886, ASSIGN because future has completed 2024-11-15T17:39:28,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d71afd5b7fbbee97ab8477168698886, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:28,362 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:39:28,363 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d71afd5b7fbbee97ab8477168698886, NAME => 'testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:28,363 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,363 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:28,363 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,363 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,365 INFO [StoreOpener-8d71afd5b7fbbee97ab8477168698886-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,368 INFO [StoreOpener-8d71afd5b7fbbee97ab8477168698886-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d71afd5b7fbbee97ab8477168698886 columnFamilyName testFamily 2024-11-15T17:39:28,368 DEBUG [StoreOpener-8d71afd5b7fbbee97ab8477168698886-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:28,369 INFO [StoreOpener-8d71afd5b7fbbee97ab8477168698886-1 {}] regionserver.HStore(327): Store=8d71afd5b7fbbee97ab8477168698886/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:28,370 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,371 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,372 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,372 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,372 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,375 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,379 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:28,380 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 8d71afd5b7fbbee97ab8477168698886; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68126678, jitterRate=0.015166610479354858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:28,380 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:39:28,381 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 8d71afd5b7fbbee97ab8477168698886: Running coprocessor pre-open hook at 1731692368363Writing region info on filesystem at 1731692368363Initializing all the Stores at 1731692368365 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692368365Cleaning up temporary data from old regions at 1731692368372 (+7 ms)Running coprocessor post-open hooks at 1731692368380 (+8 ms)Region opened successfully at 1731692368381 (+1 ms) 2024-11-15T17:39:28,383 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886., pid=15, masterSystemTime=1731692368357 2024-11-15T17:39:28,386 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:39:28,386 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:39:28,387 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8d71afd5b7fbbee97ab8477168698886, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:28,390 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d71afd5b7fbbee97ab8477168698886, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:28,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-15T17:39:28,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d71afd5b7fbbee97ab8477168698886, server=046d875b9f8f,40457,1731692313785 in 190 msec 2024-11-15T17:39:28,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T17:39:28,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=8d71afd5b7fbbee97ab8477168698886, ASSIGN in 353 msec 2024-11-15T17:39:28,404 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:28,404 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692368404"}]},"ts":"1731692368404"} 2024-11-15T17:39:28,407 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-15T17:39:28,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:28,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2140 sec 2024-11-15T17:39:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T17:39:30,228 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-15T17:39:30,228 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-15T17:39:30,228 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:30,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-15T17:39:30,234 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:30,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-15T17:39:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:30,252 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:30,252 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-15T17:39:30,256 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-15T17:39:30,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-15T17:39:30,260 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-15T17:39:30,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6272187458155184103 2024-11-15T17:39:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692373259, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:30,264 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-15T17:39:30,265 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-15T17:39:30,266 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:30,268 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:30,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6272187458155184101 2024-11-15T17:39:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692373267, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:30,272 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:30,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6272187458155184100 2024-11-15T17:39:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692373271, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:30,277 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:30,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6272187458155184099 2024-11-15T17:39:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:37102 deadline: 1731692373276, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-15T17:39:30,282 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-15T17:39:30,291 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-495086358_22 at /127.0.0.1:37338 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=149 (was 149), ProcessCount=11 (was 11), AvailableMemoryMB=1845 (was 2071) 2024-11-15T17:39:30,299 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=149, ProcessCount=11, AvailableMemoryMB=1844 2024-11-15T17:39:30,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-15T17:39:30,305 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:30,305 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:30,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-15T17:39:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-15T17:39:30,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741843_1019 (size=418) 2024-11-15T17:39:30,717 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb9a3a292b13ccbfe7868eddf637a8f9, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:30,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741844_1020 (size=69) 2024-11-15T17:39:31,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-15T17:39:31,129 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:31,130 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing eb9a3a292b13ccbfe7868eddf637a8f9, disabling compactions & flushes 2024-11-15T17:39:31,130 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:39:31,130 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:39:31,130 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. after waiting 0 ms 2024-11-15T17:39:31,130 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:39:31,130 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:39:31,130 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb9a3a292b13ccbfe7868eddf637a8f9: Waiting for close lock at 1731692371130Disabling compacts and flushes for region at 1731692371130Disabling writes for close at 1731692371130Writing region close event to WAL at 1731692371130Closed at 1731692371130 2024-11-15T17:39:31,132 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:31,133 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731692371132"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692371132"}]},"ts":"1731692371132"} 2024-11-15T17:39:31,136 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:31,138 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:31,138 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692371138"}]},"ts":"1731692371138"} 2024-11-15T17:39:31,141 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-15T17:39:31,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=eb9a3a292b13ccbfe7868eddf637a8f9, ASSIGN}] 2024-11-15T17:39:31,143 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=eb9a3a292b13ccbfe7868eddf637a8f9, ASSIGN 2024-11-15T17:39:31,145 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=eb9a3a292b13ccbfe7868eddf637a8f9, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:31,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eb9a3a292b13ccbfe7868eddf637a8f9, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:31,299 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=eb9a3a292b13ccbfe7868eddf637a8f9, ASSIGN because future has completed 2024-11-15T17:39:31,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb9a3a292b13ccbfe7868eddf637a8f9, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-15T17:39:31,460 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:39:31,460 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => eb9a3a292b13ccbfe7868eddf637a8f9, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:31,461 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,461 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:31,461 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,461 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,463 INFO [StoreOpener-eb9a3a292b13ccbfe7868eddf637a8f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,465 INFO [StoreOpener-eb9a3a292b13ccbfe7868eddf637a8f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb9a3a292b13ccbfe7868eddf637a8f9 columnFamilyName testFamily 2024-11-15T17:39:31,466 DEBUG [StoreOpener-eb9a3a292b13ccbfe7868eddf637a8f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:31,466 INFO [StoreOpener-eb9a3a292b13ccbfe7868eddf637a8f9-1 {}] regionserver.HStore(327): Store=eb9a3a292b13ccbfe7868eddf637a8f9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:31,467 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,468 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,469 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,470 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,470 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,472 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,476 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:31,477 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened eb9a3a292b13ccbfe7868eddf637a8f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66496328, jitterRate=-0.009127497673034668}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:31,477 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:39:31,478 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for eb9a3a292b13ccbfe7868eddf637a8f9: Running coprocessor pre-open hook at 1731692371461Writing region info on filesystem at 1731692371461Initializing all the Stores at 1731692371463 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692371463Cleaning up temporary data from old regions at 1731692371470 (+7 ms)Running coprocessor post-open hooks at 1731692371477 (+7 ms)Region opened successfully at 1731692371477 2024-11-15T17:39:31,479 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., pid=18, masterSystemTime=1731692371453 2024-11-15T17:39:31,482 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:39:31,482 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:39:31,483 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eb9a3a292b13ccbfe7868eddf637a8f9, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:31,486 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb9a3a292b13ccbfe7868eddf637a8f9, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:31,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T17:39:31,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure eb9a3a292b13ccbfe7868eddf637a8f9, server=046d875b9f8f,40457,1731692313785 in 188 msec 2024-11-15T17:39:31,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-15T17:39:31,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=eb9a3a292b13ccbfe7868eddf637a8f9, ASSIGN in 351 msec 2024-11-15T17:39:31,497 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:31,498 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692371497"}]},"ts":"1731692371497"} 2024-11-15T17:39:31,500 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-15T17:39:31,502 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:31,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2000 sec 2024-11-15T17:39:32,128 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:39:33,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-15T17:39:33,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:33,310 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-15T17:39:33,310 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-15T17:39:33,325 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-15T17:39:33,325 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-15T17:39:33,325 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:33,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-15T17:39:33,329 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:33,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-15T17:39:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:33,343 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:33,343 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-15T17:39:33,346 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-15T17:39:33,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-15T17:39:33,865 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:39:34,349 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=046d875b9f8f:40457 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-15T17:39:34,349 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-15T17:39:34,349 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 from cache 2024-11-15T17:39:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-15T17:39:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:35,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-15T17:39:36,349 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=046d875b9f8f:40457 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=981ms, rpcTimeout=978ms 2024-11-15T17:39:36,349 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=981ms, rpcTimeout=978ms 2024-11-15T17:39:36,349 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 from cache 2024-11-15T17:39:36,357 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 149), ProcessCount=11 (was 11), AvailableMemoryMB=1678 (was 1844) 2024-11-15T17:39:36,364 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=1678 2024-11-15T17:39:36,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-15T17:39:36,370 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:36,370 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:36,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-15T17:39:36,371 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:39:36,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741845_1021 (size=416) 2024-11-15T17:39:36,783 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ff2ee2d1a3bbd57c64c89a62b50e88f, NAME => 'testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:36,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741846_1022 (size=67) 2024-11-15T17:39:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-15T17:39:37,038 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-15T17:39:37,197 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:37,197 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 7ff2ee2d1a3bbd57c64c89a62b50e88f, disabling compactions & flushes 2024-11-15T17:39:37,197 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:39:37,197 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:39:37,197 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. after waiting 0 ms 2024-11-15T17:39:37,197 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:39:37,197 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:39:37,198 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ff2ee2d1a3bbd57c64c89a62b50e88f: Waiting for close lock at 1731692377197Disabling compacts and flushes for region at 1731692377197Disabling writes for close at 1731692377197Writing region close event to WAL at 1731692377197Closed at 1731692377197 2024-11-15T17:39:37,200 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:37,200 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731692377200"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692377200"}]},"ts":"1731692377200"} 2024-11-15T17:39:37,203 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:37,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:37,204 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692377204"}]},"ts":"1731692377204"} 2024-11-15T17:39:37,207 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-15T17:39:37,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7ff2ee2d1a3bbd57c64c89a62b50e88f, ASSIGN}] 2024-11-15T17:39:37,209 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7ff2ee2d1a3bbd57c64c89a62b50e88f, ASSIGN 2024-11-15T17:39:37,210 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7ff2ee2d1a3bbd57c64c89a62b50e88f, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:37,361 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7ff2ee2d1a3bbd57c64c89a62b50e88f, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:37,366 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7ff2ee2d1a3bbd57c64c89a62b50e88f, ASSIGN because future has completed 2024-11-15T17:39:37,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ff2ee2d1a3bbd57c64c89a62b50e88f, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:39:37,525 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:39:37,525 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ff2ee2d1a3bbd57c64c89a62b50e88f, NAME => 'testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:37,526 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,526 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:37,526 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,526 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,528 INFO [StoreOpener-7ff2ee2d1a3bbd57c64c89a62b50e88f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,531 INFO [StoreOpener-7ff2ee2d1a3bbd57c64c89a62b50e88f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ff2ee2d1a3bbd57c64c89a62b50e88f columnFamilyName testFamily 2024-11-15T17:39:37,531 DEBUG [StoreOpener-7ff2ee2d1a3bbd57c64c89a62b50e88f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:37,531 INFO [StoreOpener-7ff2ee2d1a3bbd57c64c89a62b50e88f-1 {}] regionserver.HStore(327): Store=7ff2ee2d1a3bbd57c64c89a62b50e88f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:37,532 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,533 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,534 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,534 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,534 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,537 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,540 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:37,541 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 7ff2ee2d1a3bbd57c64c89a62b50e88f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63074118, jitterRate=-0.060122400522232056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:37,541 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:39:37,542 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 7ff2ee2d1a3bbd57c64c89a62b50e88f: Running coprocessor pre-open hook at 1731692377526Writing region info on filesystem at 1731692377526Initializing all the Stores at 1731692377528 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692377528Cleaning up temporary data from old regions at 1731692377534 (+6 ms)Running coprocessor post-open hooks at 1731692377542 (+8 ms)Region opened successfully at 1731692377542 2024-11-15T17:39:37,544 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f., pid=21, masterSystemTime=1731692377520 2024-11-15T17:39:37,546 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:39:37,547 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:39:37,547 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7ff2ee2d1a3bbd57c64c89a62b50e88f, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:37,550 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ff2ee2d1a3bbd57c64c89a62b50e88f, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:37,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-15T17:39:37,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ff2ee2d1a3bbd57c64c89a62b50e88f, server=046d875b9f8f,40457,1731692313785 in 185 msec 2024-11-15T17:39:37,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T17:39:37,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7ff2ee2d1a3bbd57c64c89a62b50e88f, ASSIGN in 347 msec 2024-11-15T17:39:37,558 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:37,558 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692377558"}]},"ts":"1731692377558"} 2024-11-15T17:39:37,561 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-15T17:39:37,562 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:37,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1950 sec 2024-11-15T17:39:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T17:39:39,406 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-15T17:39:39,406 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-15T17:39:39,406 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:39,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-15T17:39:39,415 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:39,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-15T17:39:39,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:39,428 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:39,429 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-15T17:39:39,432 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-15T17:39:39,434 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-15T17:39:39,435 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-15T17:39:39,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-15T17:39:42,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184088' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-15T17:39:42,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:37102 deadline: 1731692383443, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184088' does not exist 2024-11-15T17:39:42,446 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 046d875b9f8f,40457,1731692313785 for closing scanner id = -6272187458155184088 for 7ff2ee2d1a3bbd57c64c89a62b50e88f of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184088' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:42,455 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=124 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=1645 (was 1678) 2024-11-15T17:39:42,463 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=124, ProcessCount=11, AvailableMemoryMB=1644 2024-11-15T17:39:42,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-15T17:39:42,468 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:42,468 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:42,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-15T17:39:42,470 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-15T17:39:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741847_1023 (size=411) 2024-11-15T17:39:42,881 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 388c173f002f29d820c89fdac4120d80, NAME => 'testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:42,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741848_1024 (size=62) 2024-11-15T17:39:42,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-15T17:39:43,031 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:39:43,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-15T17:39:43,296 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:43,296 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 388c173f002f29d820c89fdac4120d80, disabling compactions & flushes 2024-11-15T17:39:43,296 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:39:43,296 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:39:43,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. after waiting 1 ms 2024-11-15T17:39:43,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:39:43,297 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:39:43,297 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 388c173f002f29d820c89fdac4120d80: Waiting for close lock at 1731692383296Disabling compacts and flushes for region at 1731692383296Disabling writes for close at 1731692383297 (+1 ms)Writing region close event to WAL at 1731692383297Closed at 1731692383297 2024-11-15T17:39:43,300 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:43,300 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731692383300"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692383300"}]},"ts":"1731692383300"} 2024-11-15T17:39:43,305 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:43,307 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:43,307 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692383307"}]},"ts":"1731692383307"} 2024-11-15T17:39:43,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-15T17:39:43,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:43,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-15T17:39:43,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=388c173f002f29d820c89fdac4120d80, ASSIGN}] 2024-11-15T17:39:43,315 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=388c173f002f29d820c89fdac4120d80, ASSIGN 2024-11-15T17:39:43,317 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=388c173f002f29d820c89fdac4120d80, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:43,467 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=388c173f002f29d820c89fdac4120d80, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:43,470 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=388c173f002f29d820c89fdac4120d80, ASSIGN because future has completed 2024-11-15T17:39:43,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 388c173f002f29d820c89fdac4120d80, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-15T17:39:43,629 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:39:43,630 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 388c173f002f29d820c89fdac4120d80, NAME => 'testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:43,630 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,630 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:43,630 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,630 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,633 INFO [StoreOpener-388c173f002f29d820c89fdac4120d80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,635 INFO [StoreOpener-388c173f002f29d820c89fdac4120d80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 388c173f002f29d820c89fdac4120d80 columnFamilyName testFamily 2024-11-15T17:39:43,635 DEBUG [StoreOpener-388c173f002f29d820c89fdac4120d80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:43,636 INFO [StoreOpener-388c173f002f29d820c89fdac4120d80-1 {}] regionserver.HStore(327): Store=388c173f002f29d820c89fdac4120d80/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:43,636 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,638 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,639 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,640 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,640 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,642 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,645 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:43,646 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 388c173f002f29d820c89fdac4120d80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74348045, jitterRate=0.10787220299243927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:43,646 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:39:43,647 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 388c173f002f29d820c89fdac4120d80: Running coprocessor pre-open hook at 1731692383630Writing region info on filesystem at 1731692383631 (+1 ms)Initializing all the Stores at 1731692383632 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692383632Cleaning up temporary data from old regions at 1731692383640 (+8 ms)Running coprocessor post-open hooks at 1731692383647 (+7 ms)Region opened successfully at 1731692383647 2024-11-15T17:39:43,648 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80., pid=24, masterSystemTime=1731692383623 2024-11-15T17:39:43,651 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:39:43,651 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:39:43,652 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=388c173f002f29d820c89fdac4120d80, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:43,655 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 388c173f002f29d820c89fdac4120d80, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:43,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T17:39:43,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 388c173f002f29d820c89fdac4120d80, server=046d875b9f8f,40457,1731692313785 in 186 msec 2024-11-15T17:39:43,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-15T17:39:43,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=388c173f002f29d820c89fdac4120d80, ASSIGN in 348 msec 2024-11-15T17:39:43,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:43,666 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692383666"}]},"ts":"1731692383666"} 2024-11-15T17:39:43,669 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-15T17:39:43,670 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:43,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2060 sec 2024-11-15T17:39:45,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-15T17:39:45,495 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-15T17:39:45,496 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-15T17:39:45,496 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:45,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-15T17:39:45,503 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:45,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-15T17:39:45,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:45,517 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:45,517 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-15T17:39:45,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-15T17:39:45,523 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-15T17:39:45,524 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-15T17:39:45,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-15T17:39:47,695 INFO [master/046d875b9f8f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T17:39:47,695 INFO [master/046d875b9f8f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T17:39:48,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184084' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-15T17:39:48,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40457 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:37102 deadline: 1731692389538, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184084' does not exist 2024-11-15T17:39:48,540 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 046d875b9f8f,40457,1731692313785 for closing scanner id = -6272187458155184084 for 388c173f002f29d820c89fdac4120d80 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184084' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:39:48,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=114 (was 124), ProcessCount=11 (was 11), AvailableMemoryMB=1631 (was 1644) 2024-11-15T17:39:48,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=114, ProcessCount=11, AvailableMemoryMB=1631 2024-11-15T17:39:48,555 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:48,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-15T17:39:48,559 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:48,559 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:48,559 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-15T17:39:48,560 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:39:48,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741849_1025 (size=419) 2024-11-15T17:39:48,975 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 549fc73c4471eaada48dc52bead88b9a, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:48,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741850_1026 (size=70) 2024-11-15T17:39:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-15T17:39:49,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:39:49,165 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-15T17:39:49,388 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:49,388 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 549fc73c4471eaada48dc52bead88b9a, disabling compactions & flushes 2024-11-15T17:39:49,389 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:39:49,389 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:39:49,389 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. after waiting 0 ms 2024-11-15T17:39:49,389 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:39:49,389 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:39:49,389 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 549fc73c4471eaada48dc52bead88b9a: Waiting for close lock at 1731692389388Disabling compacts and flushes for region at 1731692389388Disabling writes for close at 1731692389389 (+1 ms)Writing region close event to WAL at 1731692389389Closed at 1731692389389 2024-11-15T17:39:49,393 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:49,394 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731692389393"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692389393"}]},"ts":"1731692389393"} 2024-11-15T17:39:49,396 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:49,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:49,398 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692389398"}]},"ts":"1731692389398"} 2024-11-15T17:39:49,400 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-15T17:39:49,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=549fc73c4471eaada48dc52bead88b9a, ASSIGN}] 2024-11-15T17:39:49,402 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=549fc73c4471eaada48dc52bead88b9a, ASSIGN 2024-11-15T17:39:49,403 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=549fc73c4471eaada48dc52bead88b9a, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:49,555 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=549fc73c4471eaada48dc52bead88b9a, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:49,561 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=549fc73c4471eaada48dc52bead88b9a, ASSIGN because future has completed 2024-11-15T17:39:49,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 549fc73c4471eaada48dc52bead88b9a, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:39:49,728 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:39:49,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 549fc73c4471eaada48dc52bead88b9a, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:49,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:49,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,729 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,731 INFO [StoreOpener-549fc73c4471eaada48dc52bead88b9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,734 INFO [StoreOpener-549fc73c4471eaada48dc52bead88b9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 549fc73c4471eaada48dc52bead88b9a columnFamilyName testFamily 2024-11-15T17:39:49,734 DEBUG [StoreOpener-549fc73c4471eaada48dc52bead88b9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:49,734 INFO [StoreOpener-549fc73c4471eaada48dc52bead88b9a-1 {}] regionserver.HStore(327): Store=549fc73c4471eaada48dc52bead88b9a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:49,735 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,736 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,737 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,737 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,738 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,744 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:49,745 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 549fc73c4471eaada48dc52bead88b9a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69890890, jitterRate=0.04145541787147522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:49,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:39:49,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 549fc73c4471eaada48dc52bead88b9a: Running coprocessor pre-open hook at 1731692389729Writing region info on filesystem at 1731692389729Initializing all the Stores at 1731692389731 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692389731Cleaning up temporary data from old regions at 1731692389738 (+7 ms)Running coprocessor post-open hooks at 1731692389745 (+7 ms)Region opened successfully at 1731692389745 2024-11-15T17:39:49,746 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a., pid=27, masterSystemTime=1731692389717 2024-11-15T17:39:49,749 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:39:49,749 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:39:49,750 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=549fc73c4471eaada48dc52bead88b9a, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:49,752 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 549fc73c4471eaada48dc52bead88b9a, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:49,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-15T17:39:49,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 549fc73c4471eaada48dc52bead88b9a, server=046d875b9f8f,40457,1731692313785 in 192 msec 2024-11-15T17:39:49,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T17:39:49,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=549fc73c4471eaada48dc52bead88b9a, ASSIGN in 356 msec 2024-11-15T17:39:49,761 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:49,761 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692389761"}]},"ts":"1731692389761"} 2024-11-15T17:39:49,764 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-15T17:39:49,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:49,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2090 sec 2024-11-15T17:39:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T17:39:51,597 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-15T17:39:51,597 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-15T17:39:51,597 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:51,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-15T17:39:51,607 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:39:51,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-15T17:39:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:39:51,621 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:39:51,621 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-15T17:39:51,624 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-15T17:39:51,625 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-15T17:39:53,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-15T17:39:53,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:53,311 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-15T17:39:53,311 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-15T17:39:55,232 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:39:55,264 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-15T17:39:57,630 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a., hostname=046d875b9f8f,40457,1731692313785, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=046d875b9f8f:40457 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-15T17:39:57,630 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-15T17:39:57,630 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 from cache 2024-11-15T17:39:57,645 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=96 (was 114), ProcessCount=11 (was 11), AvailableMemoryMB=1626 (was 1631) 2024-11-15T17:39:57,657 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=96, ProcessCount=11, AvailableMemoryMB=1626 2024-11-15T17:39:57,660 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:39:57,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-15T17:39:57,664 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:39:57,664 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:57,665 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-15T17:39:57,667 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:39:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T17:39:57,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741851_1027 (size=407) 2024-11-15T17:39:58,084 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 956f51b5b49b5e6bcfb10a42712ade16, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:39:58,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741852_1028 (size=58) 2024-11-15T17:39:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-15T17:39:58,495 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:58,496 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 956f51b5b49b5e6bcfb10a42712ade16, disabling compactions & flushes 2024-11-15T17:39:58,496 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:39:58,496 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:39:58,496 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. after waiting 0 ms 2024-11-15T17:39:58,496 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:39:58,496 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:39:58,496 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 956f51b5b49b5e6bcfb10a42712ade16: Waiting for close lock at 1731692398496Disabling compacts and flushes for region at 1731692398496Disabling writes for close at 1731692398496Writing region close event to WAL at 1731692398496Closed at 1731692398496 2024-11-15T17:39:58,499 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:39:58,500 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731692398499"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692398499"}]},"ts":"1731692398499"} 2024-11-15T17:39:58,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:39:58,506 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:39:58,507 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692398506"}]},"ts":"1731692398506"} 2024-11-15T17:39:58,511 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-15T17:39:58,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=956f51b5b49b5e6bcfb10a42712ade16, ASSIGN}] 2024-11-15T17:39:58,512 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=956f51b5b49b5e6bcfb10a42712ade16, ASSIGN 2024-11-15T17:39:58,513 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=956f51b5b49b5e6bcfb10a42712ade16, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:39:58,664 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=956f51b5b49b5e6bcfb10a42712ade16, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:58,670 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=956f51b5b49b5e6bcfb10a42712ade16, ASSIGN because future has completed 2024-11-15T17:39:58,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 956f51b5b49b5e6bcfb10a42712ade16, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:39:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T17:39:58,836 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:39:58,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 956f51b5b49b5e6bcfb10a42712ade16, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:39:58,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:39:58,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,840 INFO [StoreOpener-956f51b5b49b5e6bcfb10a42712ade16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,843 INFO [StoreOpener-956f51b5b49b5e6bcfb10a42712ade16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 956f51b5b49b5e6bcfb10a42712ade16 columnFamilyName testFamily 2024-11-15T17:39:58,843 DEBUG [StoreOpener-956f51b5b49b5e6bcfb10a42712ade16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:39:58,844 INFO [StoreOpener-956f51b5b49b5e6bcfb10a42712ade16-1 {}] regionserver.HStore(327): Store=956f51b5b49b5e6bcfb10a42712ade16/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:39:58,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,854 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:39:58,854 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 956f51b5b49b5e6bcfb10a42712ade16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71133582, jitterRate=0.05997297167778015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:39:58,855 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:39:58,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 956f51b5b49b5e6bcfb10a42712ade16: Running coprocessor pre-open hook at 1731692398838Writing region info on filesystem at 1731692398838Initializing all the Stores at 1731692398839 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692398840 (+1 ms)Cleaning up temporary data from old regions at 1731692398847 (+7 ms)Running coprocessor post-open hooks at 1731692398855 (+8 ms)Region opened successfully at 1731692398855 2024-11-15T17:39:58,857 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16., pid=30, masterSystemTime=1731692398825 2024-11-15T17:39:58,860 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:39:58,860 INFO [RS_OPEN_PRIORITY_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:39:58,861 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=956f51b5b49b5e6bcfb10a42712ade16, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:39:58,864 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 956f51b5b49b5e6bcfb10a42712ade16, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:39:58,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T17:39:58,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 956f51b5b49b5e6bcfb10a42712ade16, server=046d875b9f8f,40457,1731692313785 in 195 msec 2024-11-15T17:39:58,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T17:39:58,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=956f51b5b49b5e6bcfb10a42712ade16, ASSIGN in 358 msec 2024-11-15T17:39:58,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:39:58,873 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692398873"}]},"ts":"1731692398873"} 2024-11-15T17:39:58,875 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-15T17:39:58,876 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:39:58,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2160 sec 2024-11-15T17:40:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T17:40:00,697 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-15T17:40:00,698 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-15T17:40:00,698 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:40:00,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-15T17:40:00,709 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:40:00,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-15T17:40:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:40:00,723 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:40:00,723 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-15T17:40:00,726 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-15T17:40:00,728 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-15T17:40:00,729 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-15T17:40:00,730 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-15T17:40:01,267 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-15T17:40:02,128 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T17:40:03,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-15T17:40:03,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-15T17:40:04,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:40:06,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40457 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184076' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-15T17:40:06,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40457 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:37102 deadline: 1731692412731, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184076' does not exist 2024-11-15T17:40:06,738 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 046d875b9f8f,40457,1731692313785 for closing scanner id = -6272187458155184076 for 956f51b5b49b5e6bcfb10a42712ade16 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6272187458155184076' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T17:40:06,748 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=88 (was 96), ProcessCount=11 (was 11), AvailableMemoryMB=1623 (was 1626) 2024-11-15T17:40:06,755 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=88, ProcessCount=11, AvailableMemoryMB=1622 2024-11-15T17:40:06,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T17:40:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-15T17:40:06,759 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T17:40:06,759 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:06,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-15T17:40:06,761 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T17:40:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T17:40:06,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741853_1029 (size=423) 2024-11-15T17:40:07,174 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9ffafa8607cf6649f0381d6603c06c42, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30 2024-11-15T17:40:07,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741854_1030 (size=74) 2024-11-15T17:40:07,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-15T17:40:07,588 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:40:07,588 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9ffafa8607cf6649f0381d6603c06c42, disabling compactions & flushes 2024-11-15T17:40:07,588 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:07,588 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:07,588 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. after waiting 0 ms 2024-11-15T17:40:07,588 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:07,588 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:07,588 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9ffafa8607cf6649f0381d6603c06c42: Waiting for close lock at 1731692407588Disabling compacts and flushes for region at 1731692407588Disabling writes for close at 1731692407588Writing region close event to WAL at 1731692407588Closed at 1731692407588 2024-11-15T17:40:07,590 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T17:40:07,591 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731692407590"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731692407590"}]},"ts":"1731692407590"} 2024-11-15T17:40:07,594 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T17:40:07,596 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T17:40:07,596 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692407596"}]},"ts":"1731692407596"} 2024-11-15T17:40:07,599 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-15T17:40:07,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ffafa8607cf6649f0381d6603c06c42, ASSIGN}] 2024-11-15T17:40:07,600 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ffafa8607cf6649f0381d6603c06c42, ASSIGN 2024-11-15T17:40:07,601 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ffafa8607cf6649f0381d6603c06c42, ASSIGN; state=OFFLINE, location=046d875b9f8f,40457,1731692313785; forceNewPlan=false, retain=false 2024-11-15T17:40:07,752 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9ffafa8607cf6649f0381d6603c06c42, regionState=OPENING, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:40:07,759 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ffafa8607cf6649f0381d6603c06c42, ASSIGN because future has completed 2024-11-15T17:40:07,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ffafa8607cf6649f0381d6603c06c42, server=046d875b9f8f,40457,1731692313785}] 2024-11-15T17:40:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T17:40:07,923 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:07,923 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ffafa8607cf6649f0381d6603c06c42, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42.', STARTKEY => '', ENDKEY => ''} 2024-11-15T17:40:07,924 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,924 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T17:40:07,924 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,924 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,927 INFO [StoreOpener-9ffafa8607cf6649f0381d6603c06c42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,929 INFO [StoreOpener-9ffafa8607cf6649f0381d6603c06c42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ffafa8607cf6649f0381d6603c06c42 columnFamilyName testFamily 2024-11-15T17:40:07,930 DEBUG [StoreOpener-9ffafa8607cf6649f0381d6603c06c42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T17:40:07,930 INFO [StoreOpener-9ffafa8607cf6649f0381d6603c06c42-1 {}] regionserver.HStore(327): Store=9ffafa8607cf6649f0381d6603c06c42/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T17:40:07,931 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,932 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,933 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,933 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,933 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,936 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,940 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T17:40:07,941 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9ffafa8607cf6649f0381d6603c06c42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59947659, jitterRate=-0.1067102700471878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T17:40:07,941 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:07,941 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9ffafa8607cf6649f0381d6603c06c42: Running coprocessor pre-open hook at 1731692407925Writing region info on filesystem at 1731692407925Initializing all the Stores at 1731692407926 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731692407926Cleaning up temporary data from old regions at 1731692407933 (+7 ms)Running coprocessor post-open hooks at 1731692407941 (+8 ms)Region opened successfully at 1731692407941 2024-11-15T17:40:07,942 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., pid=33, masterSystemTime=1731692407914 2024-11-15T17:40:07,945 DEBUG [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:07,945 INFO [RS_OPEN_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:07,946 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9ffafa8607cf6649f0381d6603c06c42, regionState=OPEN, openSeqNum=2, regionLocation=046d875b9f8f,40457,1731692313785 2024-11-15T17:40:07,948 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ffafa8607cf6649f0381d6603c06c42, server=046d875b9f8f,40457,1731692313785 because future has completed 2024-11-15T17:40:07,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T17:40:07,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ffafa8607cf6649f0381d6603c06c42, server=046d875b9f8f,40457,1731692313785 in 189 msec 2024-11-15T17:40:07,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T17:40:07,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ffafa8607cf6649f0381d6603c06c42, ASSIGN in 353 msec 2024-11-15T17:40:07,955 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T17:40:07,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731692407955"}]},"ts":"1731692407955"} 2024-11-15T17:40:07,957 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-15T17:40:07,958 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T17:40:07,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2010 sec 2024-11-15T17:40:08,430 DEBUG [master/046d875b9f8f:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T17:40:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T17:40:09,798 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-15T17:40:09,798 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-15T17:40:09,799 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:40:09,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-15T17:40:09,810 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T17:40:09,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-15T17:40:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:40:09,825 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-15T17:40:09,825 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-15T17:40:09,828 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-15T17:40:09,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-15T17:40:10,840 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=046d875b9f8f:40457 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1011ms, rpcTimeout=1000ms 2024-11-15T17:40:10,840 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1011ms, rpcTimeout=1000ms 2024-11-15T17:40:10,841 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 from cache 2024-11-15T17:40:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-15T17:40:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2] 2024-11-15T17:40:11,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40457 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-15T17:40:12,830 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=046d875b9f8f:40457 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=975ms 2024-11-15T17:40:12,830 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=975ms 2024-11-15T17:40:12,830 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., hostname=046d875b9f8f,40457,1731692313785, seqNum=2 from cache 2024-11-15T17:40:12,842 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=90 (was 88) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1626 (was 1622) - AvailableMemoryMB LEAK? - 2024-11-15T17:40:12,843 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:40:12,843 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:40:12,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:12,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:12,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:12,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T17:40:12,849 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:40:12,849 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:40:12,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:12,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:12,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T17:40:12,849 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T17:40:12,850 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2119115424, stopped=false 2024-11-15T17:40:12,850 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=046d875b9f8f,40087,1731692312977 2024-11-15T17:40:12,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T17:40:12,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T17:40:12,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:40:12,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:40:12,915 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T17:40:12,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:40:12,918 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T17:40:12,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T17:40:12,918 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:40:12,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:12,919 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '046d875b9f8f,40457,1731692313785' ***** 2024-11-15T17:40:12,920 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T17:40:12,920 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T17:40:12,920 INFO [RS:0;046d875b9f8f:40457 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T17:40:12,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T17:40:12,921 INFO [RS:0;046d875b9f8f:40457 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T17:40:12,921 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 95717f6f1db1677b5c9dfc55756121bc 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 549fc73c4471eaada48dc52bead88b9a 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for b6cd4b9f27db5117c18715b0b3667bc7 2024-11-15T17:40:12,922 INFO [regionserver/046d875b9f8f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for e60caf819870f5aeb7177660cf6de15a 2024-11-15T17:40:12,922 INFO [regionserver/046d875b9f8f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 388c173f002f29d820c89fdac4120d80 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ff2ee2d1a3bbd57c64c89a62b50e88f 2024-11-15T17:40:12,922 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 95717f6f1db1677b5c9dfc55756121bc, disabling compactions & flushes 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:12,922 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 956f51b5b49b5e6bcfb10a42712ade16 2024-11-15T17:40:12,922 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:40:12,922 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. after waiting 0 ms 2024-11-15T17:40:12,922 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:12,923 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:40:12,923 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:40:12,923 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(959): stopping server 046d875b9f8f,40457,1731692313785 2024-11-15T17:40:12,923 INFO [RS:0;046d875b9f8f:40457 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T17:40:12,923 INFO [RS:0;046d875b9f8f:40457 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;046d875b9f8f:40457. 2024-11-15T17:40:12,923 DEBUG [RS:0;046d875b9f8f:40457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T17:40:12,923 DEBUG [RS:0;046d875b9f8f:40457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:12,923 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T17:40:12,923 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T17:40:12,924 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T17:40:12,924 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T17:40:12,924 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-15T17:40:12,924 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T17:40:12,924 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T17:40:12,924 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T17:40:12,924 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T17:40:12,924 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T17:40:12,924 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 549fc73c4471eaada48dc52bead88b9a, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 95717f6f1db1677b5c9dfc55756121bc, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:12,926 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 95717f6f1db1677b5c9dfc55756121bc 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:12,926 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-15T17:40:12,975 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc/.tmp/testFamily/4514859a862149bba3d37a8f98487727 is 61, key is row-0/testFamily:testQualifier/1731692361146/Put/seqid=0 2024-11-15T17:40:12,980 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/info/dc0e892db19842f686783df3112046c6 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a./info:regioninfo/1731692348658/Put/seqid=0 2024-11-15T17:40:12,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741855_1031 (size=5049) 2024-11-15T17:40:12,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741856_1032 (size=20564) 2024-11-15T17:40:13,125 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 549fc73c4471eaada48dc52bead88b9a, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 95717f6f1db1677b5c9dfc55756121bc, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:13,325 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 549fc73c4471eaada48dc52bead88b9a, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 95717f6f1db1677b5c9dfc55756121bc, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:13,356 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-15T17:40:13,386 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc/.tmp/testFamily/4514859a862149bba3d37a8f98487727 2024-11-15T17:40:13,388 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/info/dc0e892db19842f686783df3112046c6 2024-11-15T17:40:13,426 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:40:13,434 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc/.tmp/testFamily/4514859a862149bba3d37a8f98487727 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc/testFamily/4514859a862149bba3d37a8f98487727 2024-11-15T17:40:13,443 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc/testFamily/4514859a862149bba3d37a8f98487727, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:13,450 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/ns/e16bd7ea4f784dc6aa8c7043ac4fe47a is 43, key is default/ns:d/1731692347609/Put/seqid=0 2024-11-15T17:40:13,451 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 95717f6f1db1677b5c9dfc55756121bc in 524ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:13,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741857_1033 (size=5153) 2024-11-15T17:40:13,457 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-15T17:40:13,457 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNextAsync/95717f6f1db1677b5c9dfc55756121bc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:13,458 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-15T17:40:13,458 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-15T17:40:13,459 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:40:13,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-15T17:40:13,459 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 95717f6f1db1677b5c9dfc55756121bc: Waiting for close lock at 1731692412922Running coprocessor pre-close hooks at 1731692412922Disabling compacts and flushes for region at 1731692412922Disabling writes for close at 1731692412922Obtaining lock to block concurrent updates at 1731692412926 (+4 ms)Preparing flush snapshotting stores in 95717f6f1db1677b5c9dfc55756121bc at 1731692412926Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692412933 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. at 1731692412934 (+1 ms)Flushing 95717f6f1db1677b5c9dfc55756121bc/testFamily: creating writer at 1731692412936 (+2 ms)Flushing 95717f6f1db1677b5c9dfc55756121bc/testFamily: appending metadata at 1731692412970 (+34 ms)Flushing 95717f6f1db1677b5c9dfc55756121bc/testFamily: closing flushed file at 1731692412972 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f8a1f79: reopening flushed file at 1731692413433 (+461 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 95717f6f1db1677b5c9dfc55756121bc in 524ms, sequenceid=8, compaction requested=false at 1731692413451 (+18 ms)Writing region close event to WAL at 1731692413452 (+1 ms)Running coprocessor post-close hooks at 1731692413457 (+5 ms)Closed at 1731692413459 (+2 ms) 2024-11-15T17:40:13,459 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731692358085.95717f6f1db1677b5c9dfc55756121bc. 2024-11-15T17:40:13,459 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 549fc73c4471eaada48dc52bead88b9a, disabling compactions & flushes 2024-11-15T17:40:13,460 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:40:13,460 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:40:13,460 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. after waiting 0 ms 2024-11-15T17:40:13,460 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:40:13,460 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 549fc73c4471eaada48dc52bead88b9a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:13,475 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a/.tmp/testFamily/5e38fede5d99481caf70ca5263aa0ec6 is 61, key is row-0/testFamily:testQualifier/1731692391613/Put/seqid=0 2024-11-15T17:40:13,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741858_1034 (size=5049) 2024-11-15T17:40:13,526 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 549fc73c4471eaada48dc52bead88b9a, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:13,726 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 549fc73c4471eaada48dc52bead88b9a, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:13,858 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/ns/e16bd7ea4f784dc6aa8c7043ac4fe47a 2024-11-15T17:40:13,881 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a/.tmp/testFamily/5e38fede5d99481caf70ca5263aa0ec6 2024-11-15T17:40:13,889 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/table/9a4c2b6bee154835803338c9250be162 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731692348674/Put/seqid=0 2024-11-15T17:40:13,889 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a/.tmp/testFamily/5e38fede5d99481caf70ca5263aa0ec6 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a/testFamily/5e38fede5d99481caf70ca5263aa0ec6 2024-11-15T17:40:13,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741859_1035 (size=6699) 2024-11-15T17:40:13,898 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a/testFamily/5e38fede5d99481caf70ca5263aa0ec6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:13,899 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 549fc73c4471eaada48dc52bead88b9a in 439ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:13,904 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/549fc73c4471eaada48dc52bead88b9a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:13,904 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:40:13,905 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 549fc73c4471eaada48dc52bead88b9a: Waiting for close lock at 1731692413459Running coprocessor pre-close hooks at 1731692413459Disabling compacts and flushes for region at 1731692413459Disabling writes for close at 1731692413460 (+1 ms)Obtaining lock to block concurrent updates at 1731692413460Preparing flush snapshotting stores in 549fc73c4471eaada48dc52bead88b9a at 1731692413460Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692413460Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. at 1731692413461 (+1 ms)Flushing 549fc73c4471eaada48dc52bead88b9a/testFamily: creating writer at 1731692413461Flushing 549fc73c4471eaada48dc52bead88b9a/testFamily: appending metadata at 1731692413475 (+14 ms)Flushing 549fc73c4471eaada48dc52bead88b9a/testFamily: closing flushed file at 1731692413475Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48c03579: reopening flushed file at 1731692413888 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 549fc73c4471eaada48dc52bead88b9a in 439ms, sequenceid=8, compaction requested=false at 1731692413899 (+11 ms)Writing region close event to WAL at 1731692413900 (+1 ms)Running coprocessor post-close hooks at 1731692413904 (+4 ms)Closed at 1731692413904 2024-11-15T17:40:13,905 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731692388555.549fc73c4471eaada48dc52bead88b9a. 2024-11-15T17:40:13,905 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b6cd4b9f27db5117c18715b0b3667bc7, disabling compactions & flushes 2024-11-15T17:40:13,905 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:40:13,905 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:40:13,905 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. after waiting 0 ms 2024-11-15T17:40:13,905 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:40:13,905 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b6cd4b9f27db5117c18715b0b3667bc7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:13,921 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7/.tmp/testFamily/f883dbe2eaa64132a65b43280e4c2297 is 61, key is row-0/testFamily:testQualifier/1731692352044/Put/seqid=0 2024-11-15T17:40:13,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741860_1036 (size=5049) 2024-11-15T17:40:13,926 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-15T17:40:13,927 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1325): Online Regions={b6cd4b9f27db5117c18715b0b3667bc7=hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., e60caf819870f5aeb7177660cf6de15a=testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a., 1588230740=hbase:meta,,1.1588230740, 388c173f002f29d820c89fdac4120d80=testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80., 7ff2ee2d1a3bbd57c64c89a62b50e88f=testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f., eb9a3a292b13ccbfe7868eddf637a8f9=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., 956f51b5b49b5e6bcfb10a42712ade16=hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16., 9ffafa8607cf6649f0381d6603c06c42=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., 8d71afd5b7fbbee97ab8477168698886=testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.} 2024-11-15T17:40:13,927 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:14,127 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:14,295 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/table/9a4c2b6bee154835803338c9250be162 2024-11-15T17:40:14,308 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/info/dc0e892db19842f686783df3112046c6 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/info/dc0e892db19842f686783df3112046c6 2024-11-15T17:40:14,316 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/info/dc0e892db19842f686783df3112046c6, entries=100, sequenceid=56, filesize=20.1 K 2024-11-15T17:40:14,317 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/ns/e16bd7ea4f784dc6aa8c7043ac4fe47a as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/ns/e16bd7ea4f784dc6aa8c7043ac4fe47a 2024-11-15T17:40:14,325 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/ns/e16bd7ea4f784dc6aa8c7043ac4fe47a, entries=2, sequenceid=56, filesize=5.0 K 2024-11-15T17:40:14,326 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7/.tmp/testFamily/f883dbe2eaa64132a65b43280e4c2297 2024-11-15T17:40:14,327 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/.tmp/table/9a4c2b6bee154835803338c9250be162 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/table/9a4c2b6bee154835803338c9250be162 2024-11-15T17:40:14,327 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 388c173f002f29d820c89fdac4120d80, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, b6cd4b9f27db5117c18715b0b3667bc7, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:14,334 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/table/9a4c2b6bee154835803338c9250be162, entries=20, sequenceid=56, filesize=6.5 K 2024-11-15T17:40:14,334 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7/.tmp/testFamily/f883dbe2eaa64132a65b43280e4c2297 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7/testFamily/f883dbe2eaa64132a65b43280e4c2297 2024-11-15T17:40:14,335 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1411ms, sequenceid=56, compaction requested=false 2024-11-15T17:40:14,340 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-15T17:40:14,341 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T17:40:14,341 INFO [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T17:40:14,341 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731692412924Running coprocessor pre-close hooks at 1731692412924Disabling compacts and flushes for region at 1731692412924Disabling writes for close at 1731692412924Obtaining lock to block concurrent updates at 1731692412926 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1731692412926Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731692412933 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731692412934 (+1 ms)Flushing 1588230740/info: creating writer at 1731692412936 (+2 ms)Flushing 1588230740/info: appending metadata at 1731692412975 (+39 ms)Flushing 1588230740/info: closing flushed file at 1731692412975Flushing 1588230740/ns: creating writer at 1731692413433 (+458 ms)Flushing 1588230740/ns: appending metadata at 1731692413449 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731692413449Flushing 1588230740/table: creating writer at 1731692413870 (+421 ms)Flushing 1588230740/table: appending metadata at 1731692413888 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731692413888Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f8b18ca: reopening flushed file at 1731692414306 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a23e307: reopening flushed file at 1731692414316 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c98796: reopening flushed file at 1731692414325 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1411ms, sequenceid=56, compaction requested=false at 1731692414335 (+10 ms)Writing region close event to WAL at 1731692414336 (+1 ms)Running coprocessor post-close hooks at 1731692414341 (+5 ms)Closed at 1731692414341 2024-11-15T17:40:14,341 DEBUG [RS_CLOSE_META-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T17:40:14,342 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7/testFamily/f883dbe2eaa64132a65b43280e4c2297, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:14,343 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b6cd4b9f27db5117c18715b0b3667bc7 in 437ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:14,347 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnOpenScanner/b6cd4b9f27db5117c18715b0b3667bc7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:14,348 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:40:14,348 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b6cd4b9f27db5117c18715b0b3667bc7: Waiting for close lock at 1731692413905Running coprocessor pre-close hooks at 1731692413905Disabling compacts and flushes for region at 1731692413905Disabling writes for close at 1731692413905Obtaining lock to block concurrent updates at 1731692413905Preparing flush snapshotting stores in b6cd4b9f27db5117c18715b0b3667bc7 at 1731692413905Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692413905Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. at 1731692413906 (+1 ms)Flushing b6cd4b9f27db5117c18715b0b3667bc7/testFamily: creating writer at 1731692413906Flushing b6cd4b9f27db5117c18715b0b3667bc7/testFamily: appending metadata at 1731692413920 (+14 ms)Flushing b6cd4b9f27db5117c18715b0b3667bc7/testFamily: closing flushed file at 1731692413920Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30f8dbe9: reopening flushed file at 1731692414333 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b6cd4b9f27db5117c18715b0b3667bc7 in 437ms, sequenceid=8, compaction requested=false at 1731692414343 (+10 ms)Writing region close event to WAL at 1731692414343Running coprocessor post-close hooks at 1731692414348 (+5 ms)Closed at 1731692414348 2024-11-15T17:40:14,348 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731692348974.b6cd4b9f27db5117c18715b0b3667bc7. 2024-11-15T17:40:14,348 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e60caf819870f5aeb7177660cf6de15a, disabling compactions & flushes 2024-11-15T17:40:14,348 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:40:14,349 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:40:14,349 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. after waiting 0 ms 2024-11-15T17:40:14,349 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:40:14,349 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e60caf819870f5aeb7177660cf6de15a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:14,365 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a/.tmp/testFamily/4a394acfa0c549e69b356bd68238a3c7 is 61, key is row-0/testFamily:testQualifier/1731692348882/Put/seqid=0 2024-11-15T17:40:14,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741861_1037 (size=5049) 2024-11-15T17:40:14,528 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 388c173f002f29d820c89fdac4120d80, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:14,728 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 388c173f002f29d820c89fdac4120d80, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, e60caf819870f5aeb7177660cf6de15a, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:14,773 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a/.tmp/testFamily/4a394acfa0c549e69b356bd68238a3c7 2024-11-15T17:40:14,788 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a/.tmp/testFamily/4a394acfa0c549e69b356bd68238a3c7 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a/testFamily/4a394acfa0c549e69b356bd68238a3c7 2024-11-15T17:40:14,796 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a/testFamily/4a394acfa0c549e69b356bd68238a3c7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:14,797 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e60caf819870f5aeb7177660cf6de15a in 448ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:14,802 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e60caf819870f5aeb7177660cf6de15a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:14,803 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:40:14,803 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e60caf819870f5aeb7177660cf6de15a: Waiting for close lock at 1731692414348Running coprocessor pre-close hooks at 1731692414348Disabling compacts and flushes for region at 1731692414348Disabling writes for close at 1731692414349 (+1 ms)Obtaining lock to block concurrent updates at 1731692414349Preparing flush snapshotting stores in e60caf819870f5aeb7177660cf6de15a at 1731692414349Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692414349Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. at 1731692414350 (+1 ms)Flushing e60caf819870f5aeb7177660cf6de15a/testFamily: creating writer at 1731692414350Flushing e60caf819870f5aeb7177660cf6de15a/testFamily: appending metadata at 1731692414365 (+15 ms)Flushing e60caf819870f5aeb7177660cf6de15a/testFamily: closing flushed file at 1731692414365Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59d11fbe: reopening flushed file at 1731692414786 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e60caf819870f5aeb7177660cf6de15a in 448ms, sequenceid=8, compaction requested=false at 1731692414797 (+11 ms)Writing region close event to WAL at 1731692414798 (+1 ms)Running coprocessor post-close hooks at 1731692414803 (+5 ms)Closed at 1731692414803 2024-11-15T17:40:14,803 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731692347811.e60caf819870f5aeb7177660cf6de15a. 2024-11-15T17:40:14,804 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 388c173f002f29d820c89fdac4120d80, disabling compactions & flushes 2024-11-15T17:40:14,804 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:40:14,804 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:40:14,804 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. after waiting 0 ms 2024-11-15T17:40:14,804 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:40:14,804 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 388c173f002f29d820c89fdac4120d80 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:14,819 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80/.tmp/testFamily/84bb5bc6619f4b08a45eada6f74852ef is 61, key is row-0/testFamily:testQualifier/1731692385509/Put/seqid=0 2024-11-15T17:40:14,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741862_1038 (size=5049) 2024-11-15T17:40:14,928 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-15T17:40:14,928 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1325): Online Regions={388c173f002f29d820c89fdac4120d80=testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80., 7ff2ee2d1a3bbd57c64c89a62b50e88f=testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f., eb9a3a292b13ccbfe7868eddf637a8f9=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., 956f51b5b49b5e6bcfb10a42712ade16=hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16., 9ffafa8607cf6649f0381d6603c06c42=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., 8d71afd5b7fbbee97ab8477168698886=testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.} 2024-11-15T17:40:14,929 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 388c173f002f29d820c89fdac4120d80, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:15,129 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 388c173f002f29d820c89fdac4120d80, 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:15,225 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80/.tmp/testFamily/84bb5bc6619f4b08a45eada6f74852ef 2024-11-15T17:40:15,241 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80/.tmp/testFamily/84bb5bc6619f4b08a45eada6f74852ef as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80/testFamily/84bb5bc6619f4b08a45eada6f74852ef 2024-11-15T17:40:15,250 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80/testFamily/84bb5bc6619f4b08a45eada6f74852ef, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:15,251 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 388c173f002f29d820c89fdac4120d80 in 447ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:15,256 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNext/388c173f002f29d820c89fdac4120d80/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:15,257 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:40:15,257 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 388c173f002f29d820c89fdac4120d80: Waiting for close lock at 1731692414804Running coprocessor pre-close hooks at 1731692414804Disabling compacts and flushes for region at 1731692414804Disabling writes for close at 1731692414804Obtaining lock to block concurrent updates at 1731692414804Preparing flush snapshotting stores in 388c173f002f29d820c89fdac4120d80 at 1731692414804Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692414804Flushing stores of testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. at 1731692414805 (+1 ms)Flushing 388c173f002f29d820c89fdac4120d80/testFamily: creating writer at 1731692414805Flushing 388c173f002f29d820c89fdac4120d80/testFamily: appending metadata at 1731692414819 (+14 ms)Flushing 388c173f002f29d820c89fdac4120d80/testFamily: closing flushed file at 1731692414819Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44070e5: reopening flushed file at 1731692415239 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 388c173f002f29d820c89fdac4120d80 in 447ms, sequenceid=8, compaction requested=false at 1731692415252 (+13 ms)Writing region close event to WAL at 1731692415252Running coprocessor post-close hooks at 1731692415257 (+5 ms)Closed at 1731692415257 2024-11-15T17:40:15,257 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731692382465.388c173f002f29d820c89fdac4120d80. 2024-11-15T17:40:15,257 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ff2ee2d1a3bbd57c64c89a62b50e88f, disabling compactions & flushes 2024-11-15T17:40:15,257 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:40:15,257 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:40:15,257 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. after waiting 0 ms 2024-11-15T17:40:15,257 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:40:15,257 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7ff2ee2d1a3bbd57c64c89a62b50e88f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:15,273 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f/.tmp/testFamily/00331b0e1580489daff28e42e6bf4e40 is 61, key is row-0/testFamily:testQualifier/1731692379420/Put/seqid=0 2024-11-15T17:40:15,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741863_1039 (size=5049) 2024-11-15T17:40:15,329 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:15,529 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 7ff2ee2d1a3bbd57c64c89a62b50e88f, 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:15,680 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f/.tmp/testFamily/00331b0e1580489daff28e42e6bf4e40 2024-11-15T17:40:15,696 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f/.tmp/testFamily/00331b0e1580489daff28e42e6bf4e40 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f/testFamily/00331b0e1580489daff28e42e6bf4e40 2024-11-15T17:40:15,703 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f/testFamily/00331b0e1580489daff28e42e6bf4e40, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:15,705 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ff2ee2d1a3bbd57c64c89a62b50e88f in 447ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:15,709 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnNextAsync/7ff2ee2d1a3bbd57c64c89a62b50e88f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:15,710 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:40:15,710 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ff2ee2d1a3bbd57c64c89a62b50e88f: Waiting for close lock at 1731692415257Running coprocessor pre-close hooks at 1731692415257Disabling compacts and flushes for region at 1731692415257Disabling writes for close at 1731692415257Obtaining lock to block concurrent updates at 1731692415257Preparing flush snapshotting stores in 7ff2ee2d1a3bbd57c64c89a62b50e88f at 1731692415257Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692415258 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. at 1731692415258Flushing 7ff2ee2d1a3bbd57c64c89a62b50e88f/testFamily: creating writer at 1731692415258Flushing 7ff2ee2d1a3bbd57c64c89a62b50e88f/testFamily: appending metadata at 1731692415273 (+15 ms)Flushing 7ff2ee2d1a3bbd57c64c89a62b50e88f/testFamily: closing flushed file at 1731692415273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30100e11: reopening flushed file at 1731692415695 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ff2ee2d1a3bbd57c64c89a62b50e88f in 447ms, sequenceid=8, compaction requested=false at 1731692415705 (+10 ms)Writing region close event to WAL at 1731692415705Running coprocessor post-close hooks at 1731692415710 (+5 ms)Closed at 1731692415710 2024-11-15T17:40:15,710 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731692376366.7ff2ee2d1a3bbd57c64c89a62b50e88f. 2024-11-15T17:40:15,711 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb9a3a292b13ccbfe7868eddf637a8f9, disabling compactions & flushes 2024-11-15T17:40:15,711 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:40:15,711 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:40:15,711 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. after waiting 0 ms 2024-11-15T17:40:15,711 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:40:15,711 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eb9a3a292b13ccbfe7868eddf637a8f9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:15,726 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9/.tmp/testFamily/99787b142e6147c2a967d9b9a658df94 is 61, key is row-0/testFamily:testQualifier/1731692373333/Put/seqid=0 2024-11-15T17:40:15,730 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:15,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741864_1040 (size=5049) 2024-11-15T17:40:15,930 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-15T17:40:15,930 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1325): Online Regions={eb9a3a292b13ccbfe7868eddf637a8f9=testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., 956f51b5b49b5e6bcfb10a42712ade16=hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16., 9ffafa8607cf6649f0381d6603c06c42=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., 8d71afd5b7fbbee97ab8477168698886=testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.} 2024-11-15T17:40:15,930 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:16,131 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42, eb9a3a292b13ccbfe7868eddf637a8f9 2024-11-15T17:40:16,131 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9/.tmp/testFamily/99787b142e6147c2a967d9b9a658df94 2024-11-15T17:40:16,140 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9/.tmp/testFamily/99787b142e6147c2a967d9b9a658df94 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9/testFamily/99787b142e6147c2a967d9b9a658df94 2024-11-15T17:40:16,149 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9/testFamily/99787b142e6147c2a967d9b9a658df94, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:16,150 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb9a3a292b13ccbfe7868eddf637a8f9 in 439ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:16,155 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScanner/eb9a3a292b13ccbfe7868eddf637a8f9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:16,155 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:40:16,155 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb9a3a292b13ccbfe7868eddf637a8f9: Waiting for close lock at 1731692415710Running coprocessor pre-close hooks at 1731692415710Disabling compacts and flushes for region at 1731692415710Disabling writes for close at 1731692415711 (+1 ms)Obtaining lock to block concurrent updates at 1731692415711Preparing flush snapshotting stores in eb9a3a292b13ccbfe7868eddf637a8f9 at 1731692415711Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692415711Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. at 1731692415712 (+1 ms)Flushing eb9a3a292b13ccbfe7868eddf637a8f9/testFamily: creating writer at 1731692415712Flushing eb9a3a292b13ccbfe7868eddf637a8f9/testFamily: appending metadata at 1731692415725 (+13 ms)Flushing eb9a3a292b13ccbfe7868eddf637a8f9/testFamily: closing flushed file at 1731692415725Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ae838b: reopening flushed file at 1731692416139 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb9a3a292b13ccbfe7868eddf637a8f9 in 439ms, sequenceid=8, compaction requested=false at 1731692416150 (+11 ms)Writing region close event to WAL at 1731692416151 (+1 ms)Running coprocessor post-close hooks at 1731692416155 (+4 ms)Closed at 1731692416155 2024-11-15T17:40:16,156 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731692370301.eb9a3a292b13ccbfe7868eddf637a8f9. 2024-11-15T17:40:16,156 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 956f51b5b49b5e6bcfb10a42712ade16, disabling compactions & flushes 2024-11-15T17:40:16,156 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:40:16,156 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:40:16,156 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. after waiting 0 ms 2024-11-15T17:40:16,156 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:40:16,156 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 956f51b5b49b5e6bcfb10a42712ade16 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:16,170 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16/.tmp/testFamily/dc8ec7eb643447b890428583eb9bc8f7 is 61, key is row-0/testFamily:testQualifier/1731692400714/Put/seqid=0 2024-11-15T17:40:16,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741865_1041 (size=5049) 2024-11-15T17:40:16,331 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:16,531 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886, 956f51b5b49b5e6bcfb10a42712ade16, 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:16,577 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16/.tmp/testFamily/dc8ec7eb643447b890428583eb9bc8f7 2024-11-15T17:40:16,590 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16/.tmp/testFamily/dc8ec7eb643447b890428583eb9bc8f7 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16/testFamily/dc8ec7eb643447b890428583eb9bc8f7 2024-11-15T17:40:16,597 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16/testFamily/dc8ec7eb643447b890428583eb9bc8f7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:16,598 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 956f51b5b49b5e6bcfb10a42712ade16 in 442ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:16,603 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/hbase/testMetaScanTimeoutOnNext/956f51b5b49b5e6bcfb10a42712ade16/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:16,603 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:40:16,603 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 956f51b5b49b5e6bcfb10a42712ade16: Waiting for close lock at 1731692416156Running coprocessor pre-close hooks at 1731692416156Disabling compacts and flushes for region at 1731692416156Disabling writes for close at 1731692416156Obtaining lock to block concurrent updates at 1731692416156Preparing flush snapshotting stores in 956f51b5b49b5e6bcfb10a42712ade16 at 1731692416156Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692416156Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. at 1731692416157 (+1 ms)Flushing 956f51b5b49b5e6bcfb10a42712ade16/testFamily: creating writer at 1731692416157Flushing 956f51b5b49b5e6bcfb10a42712ade16/testFamily: appending metadata at 1731692416170 (+13 ms)Flushing 956f51b5b49b5e6bcfb10a42712ade16/testFamily: closing flushed file at 1731692416170Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fb075e6: reopening flushed file at 1731692416588 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 956f51b5b49b5e6bcfb10a42712ade16 in 442ms, sequenceid=8, compaction requested=false at 1731692416598 (+10 ms)Writing region close event to WAL at 1731692416598Running coprocessor post-close hooks at 1731692416603 (+5 ms)Closed at 1731692416603 2024-11-15T17:40:16,604 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731692397659.956f51b5b49b5e6bcfb10a42712ade16. 2024-11-15T17:40:16,604 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9ffafa8607cf6649f0381d6603c06c42, disabling compactions & flushes 2024-11-15T17:40:16,604 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:16,604 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:16,604 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. after waiting 0 ms 2024-11-15T17:40:16,604 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:16,604 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9ffafa8607cf6649f0381d6603c06c42 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:16,618 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42/.tmp/testFamily/64cffb1b7625457ca97c8cfaef8791d7 is 61, key is row-0/testFamily:testQualifier/1731692409815/Put/seqid=0 2024-11-15T17:40:16,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741866_1042 (size=5049) 2024-11-15T17:40:16,732 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886, 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:16,932 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-15T17:40:16,932 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1325): Online Regions={9ffafa8607cf6649f0381d6603c06c42=testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., 8d71afd5b7fbbee97ab8477168698886=testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886.} 2024-11-15T17:40:16,933 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886, 9ffafa8607cf6649f0381d6603c06c42 2024-11-15T17:40:17,026 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42/.tmp/testFamily/64cffb1b7625457ca97c8cfaef8791d7 2024-11-15T17:40:17,043 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42/.tmp/testFamily/64cffb1b7625457ca97c8cfaef8791d7 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42/testFamily/64cffb1b7625457ca97c8cfaef8791d7 2024-11-15T17:40:17,051 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42/testFamily/64cffb1b7625457ca97c8cfaef8791d7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:17,052 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9ffafa8607cf6649f0381d6603c06c42 in 448ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:17,056 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ffafa8607cf6649f0381d6603c06c42/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:17,057 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:17,057 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9ffafa8607cf6649f0381d6603c06c42: Waiting for close lock at 1731692416604Running coprocessor pre-close hooks at 1731692416604Disabling compacts and flushes for region at 1731692416604Disabling writes for close at 1731692416604Obtaining lock to block concurrent updates at 1731692416604Preparing flush snapshotting stores in 9ffafa8607cf6649f0381d6603c06c42 at 1731692416604Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692416604Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. at 1731692416605 (+1 ms)Flushing 9ffafa8607cf6649f0381d6603c06c42/testFamily: creating writer at 1731692416605Flushing 9ffafa8607cf6649f0381d6603c06c42/testFamily: appending metadata at 1731692416618 (+13 ms)Flushing 9ffafa8607cf6649f0381d6603c06c42/testFamily: closing flushed file at 1731692416618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c7c9a4c: reopening flushed file at 1731692417042 (+424 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9ffafa8607cf6649f0381d6603c06c42 in 448ms, sequenceid=8, compaction requested=false at 1731692417052 (+10 ms)Writing region close event to WAL at 1731692417053 (+1 ms)Running coprocessor post-close hooks at 1731692417057 (+4 ms)Closed at 1731692417057 2024-11-15T17:40:17,058 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731692406757.9ffafa8607cf6649f0381d6603c06c42. 2024-11-15T17:40:17,058 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d71afd5b7fbbee97ab8477168698886, disabling compactions & flushes 2024-11-15T17:40:17,058 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:40:17,058 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:40:17,058 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. after waiting 0 ms 2024-11-15T17:40:17,058 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:40:17,058 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8d71afd5b7fbbee97ab8477168698886 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-15T17:40:17,073 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886/.tmp/testFamily/c518ba9bf9ca4581b844a3b21e6086d0 is 61, key is row-0/testFamily:testQualifier/1731692370239/Put/seqid=0 2024-11-15T17:40:17,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741867_1043 (size=5049) 2024-11-15T17:40:17,133 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:40:17,333 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:40:17,479 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886/.tmp/testFamily/c518ba9bf9ca4581b844a3b21e6086d0 2024-11-15T17:40:17,534 DEBUG [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1351): Waiting on 8d71afd5b7fbbee97ab8477168698886 2024-11-15T17:40:17,544 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886/.tmp/testFamily/c518ba9bf9ca4581b844a3b21e6086d0 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886/testFamily/c518ba9bf9ca4581b844a3b21e6086d0 2024-11-15T17:40:17,552 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886/testFamily/c518ba9bf9ca4581b844a3b21e6086d0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-15T17:40:17,553 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8d71afd5b7fbbee97ab8477168698886 in 495ms, sequenceid=8, compaction requested=false 2024-11-15T17:40:17,558 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/data/default/testRetryOutOfOrderScannerNextException/8d71afd5b7fbbee97ab8477168698886/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-15T17:40:17,559 INFO [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:40:17,559 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d71afd5b7fbbee97ab8477168698886: Waiting for close lock at 1731692417058Running coprocessor pre-close hooks at 1731692417058Disabling compacts and flushes for region at 1731692417058Disabling writes for close at 1731692417058Obtaining lock to block concurrent updates at 1731692417058Preparing flush snapshotting stores in 8d71afd5b7fbbee97ab8477168698886 at 1731692417058Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731692417058Flushing stores of testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. at 1731692417059 (+1 ms)Flushing 8d71afd5b7fbbee97ab8477168698886/testFamily: creating writer at 1731692417059Flushing 8d71afd5b7fbbee97ab8477168698886/testFamily: appending metadata at 1731692417073 (+14 ms)Flushing 8d71afd5b7fbbee97ab8477168698886/testFamily: closing flushed file at 1731692417073Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60098d6e: reopening flushed file at 1731692417542 (+469 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8d71afd5b7fbbee97ab8477168698886 in 495ms, sequenceid=8, compaction requested=false at 1731692417553 (+11 ms)Writing region close event to WAL at 1731692417554 (+1 ms)Running coprocessor post-close hooks at 1731692417559 (+5 ms)Closed at 1731692417559 2024-11-15T17:40:17,559 DEBUG [RS_CLOSE_REGION-regionserver/046d875b9f8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731692367193.8d71afd5b7fbbee97ab8477168698886. 2024-11-15T17:40:17,734 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(976): stopping server 046d875b9f8f,40457,1731692313785; all regions closed. 2024-11-15T17:40:17,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741834_1010 (size=19479) 2024-11-15T17:40:17,753 DEBUG [RS:0;046d875b9f8f:40457 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/oldWALs 2024-11-15T17:40:17,753 INFO [RS:0;046d875b9f8f:40457 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 046d875b9f8f%2C40457%2C1731692313785.meta:.meta(num 1731692347424) 2024-11-15T17:40:17,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741833_1009 (size=20031) 2024-11-15T17:40:17,758 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/WALs/046d875b9f8f,40457,1731692313785/046d875b9f8f%2C40457%2C1731692313785.1731692346980 not finished, retry = 0 2024-11-15T17:40:17,867 DEBUG [RS:0;046d875b9f8f:40457 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/oldWALs 2024-11-15T17:40:17,867 INFO [RS:0;046d875b9f8f:40457 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 046d875b9f8f%2C40457%2C1731692313785:(num 1731692346980) 2024-11-15T17:40:17,867 DEBUG [RS:0;046d875b9f8f:40457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T17:40:17,868 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T17:40:17,868 INFO [RS:0;046d875b9f8f:40457 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T17:40:17,868 INFO [RS:0;046d875b9f8f:40457 {}] hbase.ChoreService(370): Chore service for: regionserver/046d875b9f8f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T17:40:17,869 INFO [RS:0;046d875b9f8f:40457 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T17:40:17,869 INFO [regionserver/046d875b9f8f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T17:40:17,871 INFO [RS:0;046d875b9f8f:40457 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40457 2024-11-15T17:40:17,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T17:40:17,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/046d875b9f8f,40457,1731692313785 2024-11-15T17:40:17,893 INFO [RS:0;046d875b9f8f:40457 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T17:40:17,904 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [046d875b9f8f,40457,1731692313785] 2024-11-15T17:40:17,914 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/046d875b9f8f,40457,1731692313785 already deleted, retry=false 2024-11-15T17:40:17,914 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 046d875b9f8f,40457,1731692313785 expired; onlineServers=0 2024-11-15T17:40:17,915 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '046d875b9f8f,40087,1731692312977' ***** 2024-11-15T17:40:17,915 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T17:40:17,915 INFO [M:0;046d875b9f8f:40087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T17:40:17,915 INFO [M:0;046d875b9f8f:40087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T17:40:17,916 DEBUG [M:0;046d875b9f8f:40087 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T17:40:17,916 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T17:40:17,916 DEBUG [M:0;046d875b9f8f:40087 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T17:40:17,916 DEBUG [master/046d875b9f8f:0:becomeActiveMaster-HFileCleaner.small.0-1731692316798 {}] cleaner.HFileCleaner(306): Exit Thread[master/046d875b9f8f:0:becomeActiveMaster-HFileCleaner.small.0-1731692316798,5,FailOnTimeoutGroup] 2024-11-15T17:40:17,916 DEBUG [master/046d875b9f8f:0:becomeActiveMaster-HFileCleaner.large.0-1731692316797 {}] cleaner.HFileCleaner(306): Exit Thread[master/046d875b9f8f:0:becomeActiveMaster-HFileCleaner.large.0-1731692316797,5,FailOnTimeoutGroup] 2024-11-15T17:40:17,917 INFO [M:0;046d875b9f8f:40087 {}] hbase.ChoreService(370): Chore service for: master/046d875b9f8f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T17:40:17,917 INFO [M:0;046d875b9f8f:40087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T17:40:17,917 DEBUG [M:0;046d875b9f8f:40087 {}] master.HMaster(1795): Stopping service threads 2024-11-15T17:40:17,918 INFO [M:0;046d875b9f8f:40087 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T17:40:17,918 INFO [M:0;046d875b9f8f:40087 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T17:40:17,919 INFO [M:0;046d875b9f8f:40087 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T17:40:17,919 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T17:40:17,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T17:40:17,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T17:40:17,925 DEBUG [M:0;046d875b9f8f:40087 {}] zookeeper.ZKUtil(347): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T17:40:17,925 WARN [M:0;046d875b9f8f:40087 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T17:40:17,926 INFO [M:0;046d875b9f8f:40087 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/.lastflushedseqids 2024-11-15T17:40:17,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741868_1044 (size=649) 2024-11-15T17:40:17,945 INFO [regionserver/046d875b9f8f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T17:40:18,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:40:18,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40457-0x1013f93b21e0001, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:40:18,005 INFO [RS:0;046d875b9f8f:40457 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T17:40:18,005 INFO [RS:0;046d875b9f8f:40457 {}] regionserver.HRegionServer(1031): Exiting; stopping=046d875b9f8f,40457,1731692313785; zookeeper connection closed. 2024-11-15T17:40:18,006 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a65d481 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a65d481 2024-11-15T17:40:18,007 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T17:40:18,341 INFO [M:0;046d875b9f8f:40087 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T17:40:18,341 INFO [M:0;046d875b9f8f:40087 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T17:40:18,342 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T17:40:18,342 INFO [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:40:18,342 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:40:18,342 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T17:40:18,342 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:40:18,342 INFO [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.02 KB heapSize=191.03 KB 2024-11-15T17:40:18,364 DEBUG [M:0;046d875b9f8f:40087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d6e4cc0bf4a6441d9a31f0cca5569494 is 82, key is hbase:meta,,1/info:regioninfo/1731692347508/Put/seqid=0 2024-11-15T17:40:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741869_1045 (size=5672) 2024-11-15T17:40:18,771 INFO [M:0;046d875b9f8f:40087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d6e4cc0bf4a6441d9a31f0cca5569494 2024-11-15T17:40:18,803 DEBUG [M:0;046d875b9f8f:40087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09af9a47fa614d92ae5350d943e4e418 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731692348681/Put/seqid=0 2024-11-15T17:40:18,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741870_1046 (size=14068) 2024-11-15T17:40:18,963 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T17:40:19,209 INFO [M:0;046d875b9f8f:40087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.47 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09af9a47fa614d92ae5350d943e4e418 2024-11-15T17:40:19,216 INFO [M:0;046d875b9f8f:40087 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 09af9a47fa614d92ae5350d943e4e418 2024-11-15T17:40:19,232 DEBUG [M:0;046d875b9f8f:40087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2dac5bdb406a4b208b817a9da4191071 is 69, key is 046d875b9f8f,40457,1731692313785/rs:state/1731692316835/Put/seqid=0 2024-11-15T17:40:19,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741871_1047 (size=5156) 2024-11-15T17:40:19,639 INFO [M:0;046d875b9f8f:40087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2dac5bdb406a4b208b817a9da4191071 2024-11-15T17:40:19,654 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d6e4cc0bf4a6441d9a31f0cca5569494 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d6e4cc0bf4a6441d9a31f0cca5569494 2024-11-15T17:40:19,661 INFO [M:0;046d875b9f8f:40087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d6e4cc0bf4a6441d9a31f0cca5569494, entries=8, sequenceid=318, filesize=5.5 K 2024-11-15T17:40:19,662 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09af9a47fa614d92ae5350d943e4e418 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09af9a47fa614d92ae5350d943e4e418 2024-11-15T17:40:19,669 INFO [M:0;046d875b9f8f:40087 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 09af9a47fa614d92ae5350d943e4e418 2024-11-15T17:40:19,669 INFO [M:0;046d875b9f8f:40087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09af9a47fa614d92ae5350d943e4e418, entries=33, sequenceid=318, filesize=13.7 K 2024-11-15T17:40:19,670 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2dac5bdb406a4b208b817a9da4191071 as hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2dac5bdb406a4b208b817a9da4191071 2024-11-15T17:40:19,676 INFO [M:0;046d875b9f8f:40087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45921/user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2dac5bdb406a4b208b817a9da4191071, entries=1, sequenceid=318, filesize=5.0 K 2024-11-15T17:40:19,677 INFO [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.02 KB/166935, heapSize ~190.73 KB/195312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1335ms, sequenceid=318, compaction requested=false 2024-11-15T17:40:19,679 INFO [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T17:40:19,679 DEBUG [M:0;046d875b9f8f:40087 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731692418342Disabling compacts and flushes for region at 1731692418342Disabling writes for close at 1731692418342Obtaining lock to block concurrent updates at 1731692418342Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731692418342Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166935, getHeapSize=195552, getOffHeapSize=0, getCellsCount=365 at 1731692418343 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731692418344 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731692418344Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731692418363 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731692418363Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731692418785 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731692418803 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731692418803Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731692419216 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731692419232 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731692419232Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12ae8eb: reopening flushed file at 1731692419652 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@458cbc9c: reopening flushed file at 1731692419661 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a192cf9: reopening flushed file at 1731692419669 (+8 ms)Finished flush of dataSize ~163.02 KB/166935, heapSize ~190.73 KB/195312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1335ms, sequenceid=318, compaction requested=false at 1731692419677 (+8 ms)Writing region close event to WAL at 1731692419679 (+2 ms)Closed at 1731692419679 2024-11-15T17:40:19,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39187 is added to blk_1073741830_1006 (size=188212) 2024-11-15T17:40:19,682 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/1399cc27-6c20-2ab9-9b22-2c90dc330f30/MasterData/WALs/046d875b9f8f,40087,1731692312977/046d875b9f8f%2C40087%2C1731692312977.1731692316090 not finished, retry = 0 2024-11-15T17:40:19,783 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T17:40:19,784 INFO [M:0;046d875b9f8f:40087 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T17:40:19,784 INFO [M:0;046d875b9f8f:40087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40087 2024-11-15T17:40:19,784 INFO [M:0;046d875b9f8f:40087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T17:40:19,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:40:19,930 INFO [M:0;046d875b9f8f:40087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T17:40:19,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40087-0x1013f93b21e0000, quorum=127.0.0.1:63121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T17:40:19,943 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@62138f9b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T17:40:19,947 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4aadda0e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T17:40:19,947 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T17:40:19,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@262167d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T17:40:19,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63308498{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/hadoop.log.dir/,STOPPED} 2024-11-15T17:40:19,951 WARN [BP-132611115-172.17.0.2-1731692309276 heartbeating to localhost/127.0.0.1:45921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T17:40:19,951 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T17:40:19,951 WARN [BP-132611115-172.17.0.2-1731692309276 heartbeating to localhost/127.0.0.1:45921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-132611115-172.17.0.2-1731692309276 (Datanode Uuid 5e4fbb00-1927-404a-863e-47921d0a9d1c) service to localhost/127.0.0.1:45921 2024-11-15T17:40:19,951 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T17:40:19,952 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef/data/data1/current/BP-132611115-172.17.0.2-1731692309276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T17:40:19,952 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/cluster_b4e550cd-5402-ac36-fa8b-7b8d305fd7ef/data/data2/current/BP-132611115-172.17.0.2-1731692309276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T17:40:19,953 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T17:40:19,959 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@275a5b49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T17:40:19,960 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6dc41719{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T17:40:19,960 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T17:40:19,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2183003e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T17:40:19,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@795e641e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f0ea9c75-f744-5cf7-aaf0-ee92cfe5828a/hadoop.log.dir/,STOPPED} 2024-11-15T17:40:19,967 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T17:40:19,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down