2024-11-17 01:55:20,291 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-17 01:55:20,302 main DEBUG Took 0.009816 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 01:55:20,303 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 01:55:20,303 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 01:55:20,310 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 01:55:20,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,317 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 01:55:20,328 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,329 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,330 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,330 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,331 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,332 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,332 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,333 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,334 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,334 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,335 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,335 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,335 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,336 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,337 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,338 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,338 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,338 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 01:55:20,339 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,339 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 01:55:20,341 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 01:55:20,342 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 01:55:20,344 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 01:55:20,344 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 01:55:20,345 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 01:55:20,346 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 01:55:20,353 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 01:55:20,356 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 01:55:20,357 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 01:55:20,357 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 01:55:20,358 main DEBUG createAppenders(={Console}) 2024-11-17 01:55:20,359 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-17 01:55:20,359 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-17 01:55:20,359 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-17 01:55:20,360 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 01:55:20,360 main DEBUG OutputStream closed 2024-11-17 01:55:20,360 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 01:55:20,360 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 01:55:20,361 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-17 01:55:20,427 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 01:55:20,429 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 01:55:20,430 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 01:55:20,431 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 01:55:20,431 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 01:55:20,431 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 01:55:20,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 01:55:20,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 01:55:20,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 01:55:20,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 01:55:20,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 01:55:20,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 01:55:20,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 01:55:20,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 01:55:20,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 01:55:20,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 01:55:20,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 01:55:20,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 01:55:20,437 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 01:55:20,438 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-17 01:55:20,438 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 01:55:20,439 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-17T01:55:20,452 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-17 01:55:20,454 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 01:55:20,455 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T01:55:20,695 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3 2024-11-17T01:55:20,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T01:55:20,735 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d, deleteOnExit=true 2024-11-17T01:55:20,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T01:55:20,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/test.cache.data in system properties and HBase conf 2024-11-17T01:55:20,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T01:55:20,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/hadoop.log.dir in system properties and HBase conf 2024-11-17T01:55:20,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T01:55:20,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T01:55:20,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T01:55:20,822 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T01:55:20,907 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T01:55:20,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T01:55:20,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T01:55:20,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T01:55:20,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T01:55:20,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T01:55:20,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T01:55:20,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T01:55:20,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T01:55:20,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T01:55:20,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/nfs.dump.dir in system properties and HBase conf 2024-11-17T01:55:20,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/java.io.tmpdir in system properties and HBase conf 2024-11-17T01:55:20,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T01:55:20,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T01:55:20,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T01:55:21,953 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T01:55:22,020 INFO [Time-limited test {}] log.Log(170): Logging initialized @2433ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T01:55:22,084 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T01:55:22,143 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T01:55:22,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T01:55:22,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T01:55:22,167 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T01:55:22,180 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T01:55:22,183 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/hadoop.log.dir/,AVAILABLE} 2024-11-17T01:55:22,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T01:55:22,418 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@562f0d0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/java.io.tmpdir/jetty-localhost-38197-hadoop-hdfs-3_4_1-tests_jar-_-any-10304587401509261190/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T01:55:22,446 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:38197} 2024-11-17T01:55:22,446 INFO [Time-limited test {}] server.Server(415): Started @2860ms 2024-11-17T01:55:23,307 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T01:55:23,317 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T01:55:23,318 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T01:55:23,319 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T01:55:23,319 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T01:55:23,320 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/hadoop.log.dir/,AVAILABLE} 2024-11-17T01:55:23,321 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T01:55:23,417 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bc78d53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/java.io.tmpdir/jetty-localhost-42093-hadoop-hdfs-3_4_1-tests_jar-_-any-620965595300776951/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T01:55:23,418 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:42093} 2024-11-17T01:55:23,418 INFO [Time-limited test {}] server.Server(415): Started @3832ms 2024-11-17T01:55:23,464 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T01:55:24,510 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d/data/data1/current/BP-1067243206-172.17.0.2-1731808521450/current, will proceed with Du for space computation calculation, 2024-11-17T01:55:24,510 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d/data/data2/current/BP-1067243206-172.17.0.2-1731808521450/current, will proceed with Du for space computation calculation, 2024-11-17T01:55:24,537 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T01:55:24,598 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4eddba81a7b2947a with lease ID 0xf8968c44277fb6f5: Processing first storage report for DS-a9874603-f4fc-4e8b-8403-03b17cf0c33d from datanode DatanodeRegistration(127.0.0.1:45353, datanodeUuid=d98683ea-0e10-498e-b4c6-87db7eceabd9, infoPort=34735, infoSecurePort=0, ipcPort=42955, storageInfo=lv=-57;cid=testClusterID;nsid=145042768;c=1731808521450) 2024-11-17T01:55:24,598 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3 2024-11-17T01:55:24,599 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4eddba81a7b2947a with lease ID 0xf8968c44277fb6f5: from storage DS-a9874603-f4fc-4e8b-8403-03b17cf0c33d node DatanodeRegistration(127.0.0.1:45353, datanodeUuid=d98683ea-0e10-498e-b4c6-87db7eceabd9, infoPort=34735, infoSecurePort=0, ipcPort=42955, storageInfo=lv=-57;cid=testClusterID;nsid=145042768;c=1731808521450), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T01:55:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4eddba81a7b2947a with lease ID 0xf8968c44277fb6f5: Processing first storage report for DS-7b90673f-05a7-45a8-8a09-8a741339a1f1 from datanode DatanodeRegistration(127.0.0.1:45353, datanodeUuid=d98683ea-0e10-498e-b4c6-87db7eceabd9, infoPort=34735, infoSecurePort=0, ipcPort=42955, storageInfo=lv=-57;cid=testClusterID;nsid=145042768;c=1731808521450) 2024-11-17T01:55:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4eddba81a7b2947a with lease ID 0xf8968c44277fb6f5: from storage DS-7b90673f-05a7-45a8-8a09-8a741339a1f1 node DatanodeRegistration(127.0.0.1:45353, datanodeUuid=d98683ea-0e10-498e-b4c6-87db7eceabd9, infoPort=34735, infoSecurePort=0, ipcPort=42955, storageInfo=lv=-57;cid=testClusterID;nsid=145042768;c=1731808521450), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T01:55:24,690 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d/zookeeper_0, clientPort=60070, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T01:55:24,702 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60070 2024-11-17T01:55:24,713 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:55:24,716 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:55:24,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741825_1001 (size=7) 2024-11-17T01:55:25,387 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 with version=8 2024-11-17T01:55:25,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/hbase-staging 2024-11-17T01:55:25,465 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T01:55:25,672 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c6c0d7a0a7c0:0 server-side Connection retries=9 2024-11-17T01:55:25,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:25,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:25,688 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:55:25,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:25,689 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:55:25,828 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T01:55:25,877 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T01:55:25,887 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T01:55:25,892 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:55:25,919 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96626 (auto-detected) 2024-11-17T01:55:25,920 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T01:55:25,941 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33177 2024-11-17T01:55:25,967 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33177 connecting to ZooKeeper ensemble=127.0.0.1:60070 2024-11-17T01:55:26,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:331770x0, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:55:26,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33177-0x10147ca8c3f0000 connected 2024-11-17T01:55:26,120 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:55:26,122 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:55:26,131 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:55:26,134 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067, hbase.cluster.distributed=false 2024-11-17T01:55:26,156 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:55:26,162 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33177 2024-11-17T01:55:26,162 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33177 2024-11-17T01:55:26,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33177 2024-11-17T01:55:26,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33177 2024-11-17T01:55:26,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33177 2024-11-17T01:55:26,272 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6c0d7a0a7c0:0 server-side Connection retries=9 2024-11-17T01:55:26,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:26,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:26,274 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T01:55:26,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T01:55:26,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T01:55:26,276 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T01:55:26,279 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T01:55:26,279 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38643 2024-11-17T01:55:26,281 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38643 connecting to ZooKeeper ensemble=127.0.0.1:60070 2024-11-17T01:55:26,283 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:55:26,285 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:55:26,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386430x0, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T01:55:26,302 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386430x0, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:55:26,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38643-0x10147ca8c3f0001 connected 2024-11-17T01:55:26,306 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T01:55:26,314 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T01:55:26,317 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T01:55:26,323 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T01:55:26,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38643 2024-11-17T01:55:26,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38643 2024-11-17T01:55:26,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38643 2024-11-17T01:55:26,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38643 2024-11-17T01:55:26,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38643 2024-11-17T01:55:26,339 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c6c0d7a0a7c0:33177 2024-11-17T01:55:26,340 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:55:26,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:55:26,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:55:26,353 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:55:26,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T01:55:26,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:26,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:26,386 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T01:55:26,387 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c6c0d7a0a7c0,33177,1731808525509 from backup master directory 2024-11-17T01:55:26,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:55:26,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:55:26,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T01:55:26,393 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:55:26,394 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:55:26,395 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T01:55:26,397 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T01:55:26,449 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/hbase.id] with ID: ee1f05f8-9043-4833-aaeb-855a9e26eac2 2024-11-17T01:55:26,449 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/.tmp/hbase.id 2024-11-17T01:55:26,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741826_1002 (size=42) 2024-11-17T01:55:26,863 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/.tmp/hbase.id]:[hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/hbase.id] 2024-11-17T01:55:26,914 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T01:55:26,919 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T01:55:26,936 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-17T01:55:26,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:26,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:27,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741827_1003 (size=196) 2024-11-17T01:55:27,435 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:55:27,436 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T01:55:27,449 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:55:27,453 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:55:27,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741828_1004 (size=1189) 2024-11-17T01:55:27,906 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store 2024-11-17T01:55:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741829_1005 (size=34) 2024-11-17T01:55:28,331 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T01:55:28,334 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:55:28,335 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:55:28,335 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:28,336 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:28,337 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:55:28,338 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:28,338 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:55:28,339 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808528335Disabling compacts and flushes for region at 1731808528335Disabling writes for close at 1731808528337 (+2 ms)Writing region close event to WAL at 1731808528338 (+1 ms)Closed at 1731808528338 2024-11-17T01:55:28,342 WARN [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/.initializing 2024-11-17T01:55:28,342 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/WALs/c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:55:28,351 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:55:28,367 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C33177%2C1731808525509, suffix=, logDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/WALs/c6c0d7a0a7c0,33177,1731808525509, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/oldWALs, maxLogs=10 2024-11-17T01:55:28,389 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/WALs/c6c0d7a0a7c0,33177,1731808525509/c6c0d7a0a7c0%2C33177%2C1731808525509.1731808528372, exclude list is [], retry=0 2024-11-17T01:55:28,405 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45353,DS-a9874603-f4fc-4e8b-8403-03b17cf0c33d,DISK] 2024-11-17T01:55:28,408 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T01:55:28,441 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/WALs/c6c0d7a0a7c0,33177,1731808525509/c6c0d7a0a7c0%2C33177%2C1731808525509.1731808528372 2024-11-17T01:55:28,442 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34735:34735)] 2024-11-17T01:55:28,443 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:55:28,443 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:55:28,446 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,447 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,484 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T01:55:28,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:28,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:28,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T01:55:28,527 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:28,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:55:28,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T01:55:28,534 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:28,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:55:28,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T01:55:28,541 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:28,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:55:28,547 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,552 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,554 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,563 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,564 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,569 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:55:28,580 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T01:55:28,589 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:55:28,592 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59706793, jitterRate=-0.11029945313930511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:55:28,603 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731808528459Initializing all the Stores at 1731808528462 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808528463 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808528463Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808528464 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808528464Cleaning up temporary data from old regions at 1731808528564 (+100 ms)Region opened successfully at 1731808528602 (+38 ms) 2024-11-17T01:55:28,605 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T01:55:28,659 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e7320, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:55:28,703 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T01:55:28,714 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T01:55:28,715 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T01:55:28,718 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T01:55:28,719 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T01:55:28,726 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-17T01:55:28,726 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T01:55:28,762 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T01:55:28,774 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T01:55:28,834 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T01:55:28,838 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T01:55:28,840 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T01:55:28,850 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T01:55:28,854 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T01:55:28,859 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T01:55:28,876 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T01:55:28,883 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T01:55:28,892 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T01:55:28,913 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T01:55:28,917 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T01:55:28,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:55:28,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T01:55:28,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:28,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:28,931 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c6c0d7a0a7c0,33177,1731808525509, sessionid=0x10147ca8c3f0000, setting cluster-up flag (Was=false) 2024-11-17T01:55:28,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:28,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:28,984 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T01:55:28,986 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:55:29,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:29,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:29,193 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T01:55:29,200 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:55:29,208 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T01:55:29,233 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(746): ClusterId : ee1f05f8-9043-4833-aaeb-855a9e26eac2 2024-11-17T01:55:29,235 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T01:55:29,270 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T01:55:29,270 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T01:55:29,275 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T01:55:29,286 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T01:55:29,294 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T01:55:29,301 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c6c0d7a0a7c0,33177,1731808525509 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T01:55:29,310 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T01:55:29,311 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dc2cb79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6c0d7a0a7c0/172.17.0.2:0 2024-11-17T01:55:29,311 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:55:29,312 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:55:29,312 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:55:29,312 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=5, maxPoolSize=5 2024-11-17T01:55:29,312 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c6c0d7a0a7c0:0, corePoolSize=10, maxPoolSize=10 2024-11-17T01:55:29,312 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,312 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:55:29,312 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,317 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731808559317 2024-11-17T01:55:29,317 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:55:29,318 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T01:55:29,319 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T01:55:29,320 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T01:55:29,323 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:29,323 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T01:55:29,325 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T01:55:29,325 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T01:55:29,325 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T01:55:29,326 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T01:55:29,326 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c6c0d7a0a7c0:38643 2024-11-17T01:55:29,326 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,331 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T01:55:29,331 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T01:55:29,331 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T01:55:29,331 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T01:55:29,332 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T01:55:29,333 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T01:55:29,334 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6c0d7a0a7c0,33177,1731808525509 with port=38643, startcode=1731808526242 2024-11-17T01:55:29,335 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T01:55:29,335 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T01:55:29,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741831_1007 (size=1321) 2024-11-17T01:55:29,338 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808529337,5,FailOnTimeoutGroup] 2024-11-17T01:55:29,339 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808529339,5,FailOnTimeoutGroup] 2024-11-17T01:55:29,339 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,339 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T01:55:29,340 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,341 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,344 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T01:55:29,377 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T01:55:29,383 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33177 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:55:29,385 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33177 {}] master.ServerManager(517): Registering regionserver=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:55:29,400 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:55:29,400 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40955 2024-11-17T01:55:29,400 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T01:55:29,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:55:29,426 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] zookeeper.ZKUtil(111): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:55:29,426 WARN [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T01:55:29,426 INFO [RS:0;c6c0d7a0a7c0:38643 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:55:29,426 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:55:29,430 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6c0d7a0a7c0,38643,1731808526242] 2024-11-17T01:55:29,460 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T01:55:29,479 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T01:55:29,484 INFO [RS:0;c6c0d7a0a7c0:38643 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T01:55:29,485 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,486 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T01:55:29,491 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T01:55:29,492 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,492 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,492 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,492 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,492 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,493 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,493 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=2, maxPoolSize=2 2024-11-17T01:55:29,493 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,493 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,493 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,493 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,494 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,494 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6c0d7a0a7c0:0, corePoolSize=1, maxPoolSize=1 2024-11-17T01:55:29,494 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:55:29,494 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6c0d7a0a7c0:0, corePoolSize=3, maxPoolSize=3 2024-11-17T01:55:29,495 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,495 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,495 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,495 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,495 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,496 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38643,1731808526242-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:55:29,516 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T01:55:29,518 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,38643,1731808526242-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,518 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,519 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.Replication(171): c6c0d7a0a7c0,38643,1731808526242 started 2024-11-17T01:55:29,537 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:55:29,538 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1482): Serving as c6c0d7a0a7c0,38643,1731808526242, RpcServer on c6c0d7a0a7c0/172.17.0.2:38643, sessionid=0x10147ca8c3f0001 2024-11-17T01:55:29,539 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T01:55:29,539 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:55:29,539 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,38643,1731808526242' 2024-11-17T01:55:29,539 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T01:55:29,540 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T01:55:29,541 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T01:55:29,541 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T01:55:29,541 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:55:29,541 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6c0d7a0a7c0,38643,1731808526242' 2024-11-17T01:55:29,541 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T01:55:29,542 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T01:55:29,543 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T01:55:29,543 INFO [RS:0;c6c0d7a0a7c0:38643 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T01:55:29,543 INFO [RS:0;c6c0d7a0a7c0:38643 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T01:55:29,742 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T01:55:29,743 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:55:29,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741832_1008 (size=32) 2024-11-17T01:55:30,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:55:30,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T01:55:30,167 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:55:30,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:30,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:30,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T01:55:30,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:55:30,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:30,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:30,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T01:55:30,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:55:30,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:30,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:30,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T01:55:30,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:55:30,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:30,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:30,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:55:30,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740 2024-11-17T01:55:30,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740 2024-11-17T01:55:30,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:55:30,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:55:30,185 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:55:30,187 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:55:30,191 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:55:30,192 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67432123, jitterRate=0.0048169344663619995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:55:30,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731808530160Initializing all the Stores at 1731808530164 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808530164Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808530165 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808530165Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808530165Cleaning up temporary data from old regions at 1731808530184 (+19 ms)Region opened successfully at 1731808530194 (+10 ms) 2024-11-17T01:55:30,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:55:30,194 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:55:30,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:55:30,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:55:30,195 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:55:30,196 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:55:30,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808530194Disabling compacts and flushes for region at 1731808530194Disabling writes for close at 1731808530194Writing region close event to WAL at 1731808530195 (+1 ms)Closed at 1731808530196 (+1 ms) 2024-11-17T01:55:30,199 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:55:30,199 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T01:55:30,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T01:55:30,214 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:55:30,219 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T01:55:30,370 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:31,371 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:32,372 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:33,373 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:34,374 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:35,375 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:35,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T01:55:35,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T01:55:36,376 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:37,376 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:38,377 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:39,378 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:40,379 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:41,380 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:42,381 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:43,381 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:44,382 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:45,383 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:46,384 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:47,385 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:48,385 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:49,386 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:50,387 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:51,388 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:52,389 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:53,390 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:54,392 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:54,594 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:55:55,393 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:56,394 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:57,395 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:58,396 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:59,396 WARN [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T01:55:59,551 INFO [RS:0;c6c0d7a0a7c0:38643 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T01:55:59,556 INFO [RS:0;c6c0d7a0a7c0:38643 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C38643%2C1731808526242, suffix=, logDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/oldWALs, maxLogs=32 2024-11-17T01:55:59,572 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242/c6c0d7a0a7c0%2C38643%2C1731808526242.1731808559559, exclude list is [], retry=0 2024-11-17T01:55:59,576 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45353,DS-a9874603-f4fc-4e8b-8403-03b17cf0c33d,DISK] 2024-11-17T01:55:59,580 INFO [RS:0;c6c0d7a0a7c0:38643 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242/c6c0d7a0a7c0%2C38643%2C1731808526242.1731808559559 2024-11-17T01:55:59,580 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34735:34735)] 2024-11-17T01:55:59,650 DEBUG [c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T01:55:59,664 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:55:59,670 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,38643,1731808526242, state=OPENING 2024-11-17T01:55:59,713 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T01:55:59,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:59,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:55:59,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:55:59,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:55:59,725 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T01:55:59,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:55:59,903 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T01:55:59,906 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T01:55:59,917 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T01:55:59,918 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T01:55:59,918 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T01:55:59,921 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6c0d7a0a7c0%2C38643%2C1731808526242.meta, suffix=.meta, logDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/oldWALs, maxLogs=32 2024-11-17T01:55:59,939 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242/c6c0d7a0a7c0%2C38643%2C1731808526242.meta.1731808559924.meta, exclude list is [], retry=0 2024-11-17T01:55:59,943 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45353,DS-a9874603-f4fc-4e8b-8403-03b17cf0c33d,DISK] 2024-11-17T01:55:59,946 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242/c6c0d7a0a7c0%2C38643%2C1731808526242.meta.1731808559924.meta 2024-11-17T01:55:59,946 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34735:34735)] 2024-11-17T01:55:59,946 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:55:59,948 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T01:55:59,950 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T01:55:59,955 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T01:55:59,960 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T01:55:59,961 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:55:59,961 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T01:55:59,961 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T01:55:59,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T01:55:59,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T01:55:59,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:59,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:59,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T01:55:59,969 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T01:55:59,969 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:59,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:59,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T01:55:59,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T01:55:59,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:59,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:59,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T01:55:59,974 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T01:55:59,974 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:55:59,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T01:55:59,975 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T01:55:59,977 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740 2024-11-17T01:55:59,980 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740 2024-11-17T01:55:59,983 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T01:55:59,983 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T01:55:59,984 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T01:55:59,987 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T01:55:59,989 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63541281, jitterRate=-0.0531611293554306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T01:55:59,989 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T01:55:59,991 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731808559962Writing region info on filesystem at 1731808559962Initializing all the Stores at 1731808559964 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808559964Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808559964Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808559965 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731808559965Cleaning up temporary data from old regions at 1731808559983 (+18 ms)Running coprocessor post-open hooks at 1731808559989 (+6 ms)Region opened successfully at 1731808559991 (+2 ms) 2024-11-17T01:55:59,999 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731808559896 2024-11-17T01:56:00,011 DEBUG [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T01:56:00,011 INFO [RS_OPEN_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T01:56:00,013 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:00,016 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6c0d7a0a7c0,38643,1731808526242, state=OPEN 2024-11-17T01:56:00,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:56:00,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T01:56:00,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:56:00,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T01:56:00,062 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:00,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T01:56:00,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6c0d7a0a7c0,38643,1731808526242 in 334 msec 2024-11-17T01:56:00,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T01:56:00,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8640 sec 2024-11-17T01:56:00,078 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T01:56:00,078 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T01:56:00,096 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:56:00,098 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=-1] 2024-11-17T01:56:00,118 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:56:00,121 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43565, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:56:00,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9030 sec 2024-11-17T01:56:00,141 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731808560141, completionTime=-1 2024-11-17T01:56:00,144 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T01:56:00,145 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T01:56:00,174 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-17T01:56:00,174 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731808620174 2024-11-17T01:56:00,174 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731808680174 2024-11-17T01:56:00,174 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-17T01:56:00,176 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33177,1731808525509-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T01:56:00,176 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33177,1731808525509-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:56:00,176 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33177,1731808525509-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:56:00,177 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c6c0d7a0a7c0:33177, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:56:00,177 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T01:56:00,178 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T01:56:00,185 DEBUG [master/c6c0d7a0a7c0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T01:56:00,203 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.809sec 2024-11-17T01:56:00,204 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T01:56:00,205 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T01:56:00,206 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T01:56:00,206 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T01:56:00,206 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T01:56:00,207 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33177,1731808525509-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T01:56:00,207 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33177,1731808525509-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T01:56:00,214 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T01:56:00,215 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T01:56:00,215 INFO [master/c6c0d7a0a7c0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6c0d7a0a7c0,33177,1731808525509-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T01:56:00,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54dc6f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:56:00,283 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T01:56:00,284 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T01:56:00,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,33177,-1 for getting cluster id 2024-11-17T01:56:00,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:56:00,296 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee1f05f8-9043-4833-aaeb-855a9e26eac2' 2024-11-17T01:56:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:56:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee1f05f8-9043-4833-aaeb-855a9e26eac2" 2024-11-17T01:56:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18214de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:56:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,33177,-1] 2024-11-17T01:56:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:56:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:56:00,307 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:56:00,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f0683d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:56:00,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:56:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=-1] 2024-11-17T01:56:00,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:56:00,321 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:56:00,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:56:00,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1a8e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:56:00,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6c0d7a0a7c0,33177,-1 for getting cluster id 2024-11-17T01:56:00,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T01:56:00,342 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ee1f05f8-9043-4833-aaeb-855a9e26eac2' 2024-11-17T01:56:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T01:56:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ee1f05f8-9043-4833-aaeb-855a9e26eac2" 2024-11-17T01:56:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9487746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:56:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6c0d7a0a7c0,33177,-1] 2024-11-17T01:56:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T01:56:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:56:00,346 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T01:56:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4fe4b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T01:56:00,359 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=6281 2024-11-17T01:56:00,365 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T01:56:00,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:56:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@494ab692 2024-11-17T01:56:00,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T01:56:00,392 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T01:56:00,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-17T01:56:00,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:00,410 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-17T01:56:00,411 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:00,414 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:56:00,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741835_1011 (size=428) 2024-11-17T01:56:00,835 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f7df8e6f82328512994f55ec7a732b7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:00,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741836_1012 (size=79) 2024-11-17T01:56:00,847 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:00,847 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 1f7df8e6f82328512994f55ec7a732b7, disabling compactions & flushes 2024-11-17T01:56:00,847 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:56:00,847 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:56:00,847 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. after waiting 0 ms 2024-11-17T01:56:00,847 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:56:00,847 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:56:00,847 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f7df8e6f82328512994f55ec7a732b7: Waiting for close lock at 1731808560847Disabling compacts and flushes for region at 1731808560847Disabling writes for close at 1731808560847Writing region close event to WAL at 1731808560847Closed at 1731808560847 2024-11-17T01:56:00,849 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:00,854 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731808560850"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808560850"}]},"ts":"1731808560850"} 2024-11-17T01:56:00,858 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:00,860 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:00,863 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808560860"}]},"ts":"1731808560860"} 2024-11-17T01:56:00,867 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-17T01:56:00,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1f7df8e6f82328512994f55ec7a732b7, ASSIGN}] 2024-11-17T01:56:00,871 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1f7df8e6f82328512994f55ec7a732b7, ASSIGN 2024-11-17T01:56:00,873 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1f7df8e6f82328512994f55ec7a732b7, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:01,026 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f7df8e6f82328512994f55ec7a732b7, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:01,035 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1f7df8e6f82328512994f55ec7a732b7, ASSIGN because future has completed 2024-11-17T01:56:01,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f7df8e6f82328512994f55ec7a732b7, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:01,205 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:56:01,205 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f7df8e6f82328512994f55ec7a732b7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:01,206 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,206 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:01,206 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,206 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,208 INFO [StoreOpener-1f7df8e6f82328512994f55ec7a732b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,211 INFO [StoreOpener-1f7df8e6f82328512994f55ec7a732b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f7df8e6f82328512994f55ec7a732b7 columnFamilyName testFamily 2024-11-17T01:56:01,211 DEBUG [StoreOpener-1f7df8e6f82328512994f55ec7a732b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:01,212 INFO [StoreOpener-1f7df8e6f82328512994f55ec7a732b7-1 {}] regionserver.HStore(327): Store=1f7df8e6f82328512994f55ec7a732b7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:01,212 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,214 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,215 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,215 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,215 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,218 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,222 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:01,223 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1f7df8e6f82328512994f55ec7a732b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68149258, jitterRate=0.015503078699111938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:01,223 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:56:01,224 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1f7df8e6f82328512994f55ec7a732b7: Running coprocessor pre-open hook at 1731808561206Writing region info on filesystem at 1731808561206Initializing all the Stores at 1731808561208 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808561208Cleaning up temporary data from old regions at 1731808561215 (+7 ms)Running coprocessor post-open hooks at 1731808561223 (+8 ms)Region opened successfully at 1731808561224 (+1 ms) 2024-11-17T01:56:01,226 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7., pid=6, masterSystemTime=1731808561191 2024-11-17T01:56:01,229 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:56:01,229 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:56:01,230 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f7df8e6f82328512994f55ec7a732b7, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:01,234 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f7df8e6f82328512994f55ec7a732b7, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:01,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T01:56:01,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f7df8e6f82328512994f55ec7a732b7, server=c6c0d7a0a7c0,38643,1731808526242 in 200 msec 2024-11-17T01:56:01,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T01:56:01,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1f7df8e6f82328512994f55ec7a732b7, ASSIGN in 370 msec 2024-11-17T01:56:01,245 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:01,245 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808561245"}]},"ts":"1731808561245"} 2024-11-17T01:56:01,249 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-17T01:56:01,251 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:01,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 851 msec 2024-11-17T01:56:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T01:56:01,439 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-17T01:56:01,439 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-17T01:56:01,442 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:01,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-17T01:56:01,451 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:01,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-17T01:56:01,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T01:56:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=-1] 2024-11-17T01:56:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T01:56:01,459 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T01:56:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:01,478 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:01,482 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-17T01:56:01,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-17T01:56:01,491 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-17T01:56:01,493 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-17T01:56:01,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=207735104112427016 2024-11-17T01:56:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808564492, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:01,518 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-17T01:56:01,520 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-17T01:56:01,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:01,523 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:01,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=207735104112427018 2024-11-17T01:56:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808564522, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:01,527 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:01,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=207735104112427019 2024-11-17T01:56:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808564527, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:01,532 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:01,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=207735104112427020 2024-11-17T01:56:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808564531, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:01,536 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:01,547 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1157062217_22 at /127.0.0.1:43750 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 214), ProcessCount=11 (was 11), AvailableMemoryMB=6241 (was 6281) 2024-11-17T01:56:01,555 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=6241 2024-11-17T01:56:01,557 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:01,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-17T01:56:01,561 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:01,561 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:01,562 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-17T01:56:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:56:01,563 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:01,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741837_1013 (size=414) 2024-11-17T01:56:01,979 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f7c51a52b68bf2ed5bf01e5bd055060, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741838_1014 (size=65) 2024-11-17T01:56:02,392 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:02,392 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 8f7c51a52b68bf2ed5bf01e5bd055060, disabling compactions & flushes 2024-11-17T01:56:02,393 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:56:02,393 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:56:02,393 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. after waiting 0 ms 2024-11-17T01:56:02,393 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:56:02,393 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:56:02,393 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f7c51a52b68bf2ed5bf01e5bd055060: Waiting for close lock at 1731808562392Disabling compacts and flushes for region at 1731808562392Disabling writes for close at 1731808562393 (+1 ms)Writing region close event to WAL at 1731808562393Closed at 1731808562393 2024-11-17T01:56:02,397 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:02,398 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731808562398"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808562398"}]},"ts":"1731808562398"} 2024-11-17T01:56:02,403 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:02,404 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:02,405 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808562404"}]},"ts":"1731808562404"} 2024-11-17T01:56:02,408 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-17T01:56:02,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=8f7c51a52b68bf2ed5bf01e5bd055060, ASSIGN}] 2024-11-17T01:56:02,410 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=8f7c51a52b68bf2ed5bf01e5bd055060, ASSIGN 2024-11-17T01:56:02,412 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=8f7c51a52b68bf2ed5bf01e5bd055060, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:02,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8f7c51a52b68bf2ed5bf01e5bd055060, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:02,566 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=8f7c51a52b68bf2ed5bf01e5bd055060, ASSIGN because future has completed 2024-11-17T01:56:02,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f7c51a52b68bf2ed5bf01e5bd055060, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:56:02,732 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:56:02,732 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f7c51a52b68bf2ed5bf01e5bd055060, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:02,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:02,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,737 INFO [StoreOpener-8f7c51a52b68bf2ed5bf01e5bd055060-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,739 INFO [StoreOpener-8f7c51a52b68bf2ed5bf01e5bd055060-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f7c51a52b68bf2ed5bf01e5bd055060 columnFamilyName testFamily 2024-11-17T01:56:02,739 DEBUG [StoreOpener-8f7c51a52b68bf2ed5bf01e5bd055060-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:02,740 INFO [StoreOpener-8f7c51a52b68bf2ed5bf01e5bd055060-1 {}] regionserver.HStore(327): Store=8f7c51a52b68bf2ed5bf01e5bd055060/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:02,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:02,751 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 8f7c51a52b68bf2ed5bf01e5bd055060; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65384070, jitterRate=-0.025701433420181274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:02,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:56:02,752 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8f7c51a52b68bf2ed5bf01e5bd055060: Running coprocessor pre-open hook at 1731808562734Writing region info on filesystem at 1731808562734Initializing all the Stores at 1731808562736 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808562736Cleaning up temporary data from old regions at 1731808562743 (+7 ms)Running coprocessor post-open hooks at 1731808562751 (+8 ms)Region opened successfully at 1731808562752 (+1 ms) 2024-11-17T01:56:02,753 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060., pid=9, masterSystemTime=1731808562720 2024-11-17T01:56:02,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:56:02,757 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:56:02,758 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8f7c51a52b68bf2ed5bf01e5bd055060, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:02,761 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f7c51a52b68bf2ed5bf01e5bd055060, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:02,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-17T01:56:02,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f7c51a52b68bf2ed5bf01e5bd055060, server=c6c0d7a0a7c0,38643,1731808526242 in 196 msec 2024-11-17T01:56:02,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T01:56:02,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=8f7c51a52b68bf2ed5bf01e5bd055060, ASSIGN in 358 msec 2024-11-17T01:56:02,771 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:02,771 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808562771"}]},"ts":"1731808562771"} 2024-11-17T01:56:02,773 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-17T01:56:02,775 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:02,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2180 sec 2024-11-17T01:56:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T01:56:04,597 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-17T01:56:04,597 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-17T01:56:04,597 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:04,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-17T01:56:04,605 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:04,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-17T01:56:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:04,621 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:04,622 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-17T01:56:04,625 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-17T01:56:04,627 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-17T01:56:05,501 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:56:05,534 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-17T01:56:05,536 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T01:56:05,537 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-17T01:56:05,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T01:56:05,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:05,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:56:05,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T01:56:05,880 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-17T01:56:05,880 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:05,883 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-17T01:56:05,883 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:10,650 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=c6c0d7a0a7c0:38643 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6022ms, rpcTimeout=5999ms 2024-11-17T01:56:10,650 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6022ms, rpcTimeout=5999ms 2024-11-17T01:56:10,651 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 from cache 2024-11-17T01:56:10,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=181 (was 214), ProcessCount=11 (was 11), AvailableMemoryMB=6225 (was 6241) 2024-11-17T01:56:10,669 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=181, ProcessCount=11, AvailableMemoryMB=6225 2024-11-17T01:56:10,671 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:10,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-17T01:56:10,675 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:10,675 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:10,675 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-17T01:56:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-17T01:56:10,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:10,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741839_1015 (size=412) 2024-11-17T01:56:11,090 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ac2a5bf7f225df52619c184f8fa701c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:11,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741840_1016 (size=63) 2024-11-17T01:56:11,128 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-17T01:56:11,504 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:11,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 8ac2a5bf7f225df52619c184f8fa701c, disabling compactions & flushes 2024-11-17T01:56:11,505 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:56:11,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:56:11,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. after waiting 0 ms 2024-11-17T01:56:11,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:56:11,505 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:56:11,506 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ac2a5bf7f225df52619c184f8fa701c: Waiting for close lock at 1731808571504Disabling compacts and flushes for region at 1731808571504Disabling writes for close at 1731808571505 (+1 ms)Writing region close event to WAL at 1731808571505Closed at 1731808571505 2024-11-17T01:56:11,510 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:11,511 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731808571510"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808571510"}]},"ts":"1731808571510"} 2024-11-17T01:56:11,516 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:11,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:11,517 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808571517"}]},"ts":"1731808571517"} 2024-11-17T01:56:11,520 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-17T01:56:11,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=8ac2a5bf7f225df52619c184f8fa701c, ASSIGN}] 2024-11-17T01:56:11,522 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=8ac2a5bf7f225df52619c184f8fa701c, ASSIGN 2024-11-17T01:56:11,524 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=8ac2a5bf7f225df52619c184f8fa701c, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:11,675 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8ac2a5bf7f225df52619c184f8fa701c, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:11,683 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=8ac2a5bf7f225df52619c184f8fa701c, ASSIGN because future has completed 2024-11-17T01:56:11,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac2a5bf7f225df52619c184f8fa701c, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-17T01:56:11,851 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:56:11,852 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ac2a5bf7f225df52619c184f8fa701c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:11,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:11,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,855 INFO [StoreOpener-8ac2a5bf7f225df52619c184f8fa701c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,857 INFO [StoreOpener-8ac2a5bf7f225df52619c184f8fa701c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ac2a5bf7f225df52619c184f8fa701c columnFamilyName testFamily 2024-11-17T01:56:11,857 DEBUG [StoreOpener-8ac2a5bf7f225df52619c184f8fa701c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:11,857 INFO [StoreOpener-8ac2a5bf7f225df52619c184f8fa701c-1 {}] regionserver.HStore(327): Store=8ac2a5bf7f225df52619c184f8fa701c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:11,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,860 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,860 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,865 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:11,866 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 8ac2a5bf7f225df52619c184f8fa701c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68221995, jitterRate=0.016586944460868835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:11,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:56:11,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 8ac2a5bf7f225df52619c184f8fa701c: Running coprocessor pre-open hook at 1731808571853Writing region info on filesystem at 1731808571853Initializing all the Stores at 1731808571854 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808571854Cleaning up temporary data from old regions at 1731808571860 (+6 ms)Running coprocessor post-open hooks at 1731808571867 (+7 ms)Region opened successfully at 1731808571867 2024-11-17T01:56:11,869 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c., pid=12, masterSystemTime=1731808571840 2024-11-17T01:56:11,872 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:56:11,872 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:56:11,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8ac2a5bf7f225df52619c184f8fa701c, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:11,875 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ac2a5bf7f225df52619c184f8fa701c, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:11,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T01:56:11,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ac2a5bf7f225df52619c184f8fa701c, server=c6c0d7a0a7c0,38643,1731808526242 in 194 msec 2024-11-17T01:56:11,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-17T01:56:11,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=8ac2a5bf7f225df52619c184f8fa701c, ASSIGN in 361 msec 2024-11-17T01:56:11,885 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:11,885 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808571885"}]},"ts":"1731808571885"} 2024-11-17T01:56:11,888 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-17T01:56:11,889 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:11,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2180 sec 2024-11-17T01:56:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-17T01:56:13,698 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-17T01:56:13,698 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-17T01:56:13,698 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:13,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-17T01:56:13,708 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:13,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-17T01:56:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:13,722 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:13,723 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-17T01:56:13,726 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-17T01:56:13,728 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-17T01:56:13,730 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-17T01:56:13,731 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-17T01:56:15,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-17T01:56:15,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:17,393 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:56:17,540 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-17T01:56:19,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38643 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427029' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-17T01:56:19,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:52434 deadline: 1731808585738, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427029' does not exist 2024-11-17T01:56:19,745 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to c6c0d7a0a7c0,38643,1731808526242 for closing scanner id = 207735104112427029 for 8ac2a5bf7f225df52619c184f8fa701c of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427029' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:56:19,748 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=161 (was 181), ProcessCount=11 (was 11), AvailableMemoryMB=6212 (was 6225) 2024-11-17T01:56:19,756 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=161, ProcessCount=11, AvailableMemoryMB=6211 2024-11-17T01:56:19,758 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-17T01:56:19,761 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:19,761 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:19,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-17T01:56:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:56:19,763 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:19,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741841_1017 (size=423) 2024-11-17T01:56:20,179 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 850615d431cafffcd9ff2c59be97ff94, NAME => 'testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:20,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741842_1018 (size=74) 2024-11-17T01:56:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-17T01:56:20,595 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:20,595 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 850615d431cafffcd9ff2c59be97ff94, disabling compactions & flushes 2024-11-17T01:56:20,595 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:56:20,595 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:56:20,595 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. after waiting 0 ms 2024-11-17T01:56:20,596 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:56:20,596 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:56:20,596 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 850615d431cafffcd9ff2c59be97ff94: Waiting for close lock at 1731808580595Disabling compacts and flushes for region at 1731808580595Disabling writes for close at 1731808580595Writing region close event to WAL at 1731808580596 (+1 ms)Closed at 1731808580596 2024-11-17T01:56:20,600 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:20,601 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731808580600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808580600"}]},"ts":"1731808580600"} 2024-11-17T01:56:20,606 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:20,608 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:20,608 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808580608"}]},"ts":"1731808580608"} 2024-11-17T01:56:20,610 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-17T01:56:20,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=850615d431cafffcd9ff2c59be97ff94, ASSIGN}] 2024-11-17T01:56:20,612 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=850615d431cafffcd9ff2c59be97ff94, ASSIGN 2024-11-17T01:56:20,614 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=850615d431cafffcd9ff2c59be97ff94, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:20,765 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=850615d431cafffcd9ff2c59be97ff94, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:20,772 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=850615d431cafffcd9ff2c59be97ff94, ASSIGN because future has completed 2024-11-17T01:56:20,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 850615d431cafffcd9ff2c59be97ff94, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:56:20,938 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:56:20,939 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 850615d431cafffcd9ff2c59be97ff94, NAME => 'testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:20,939 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,939 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:20,940 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,940 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,943 INFO [StoreOpener-850615d431cafffcd9ff2c59be97ff94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,946 INFO [StoreOpener-850615d431cafffcd9ff2c59be97ff94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 850615d431cafffcd9ff2c59be97ff94 columnFamilyName testFamily 2024-11-17T01:56:20,946 DEBUG [StoreOpener-850615d431cafffcd9ff2c59be97ff94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:20,947 INFO [StoreOpener-850615d431cafffcd9ff2c59be97ff94-1 {}] regionserver.HStore(327): Store=850615d431cafffcd9ff2c59be97ff94/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:20,947 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,948 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,948 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,949 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,949 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,951 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,955 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:20,955 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 850615d431cafffcd9ff2c59be97ff94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75478442, jitterRate=0.12471643090248108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:20,956 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:56:20,956 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 850615d431cafffcd9ff2c59be97ff94: Running coprocessor pre-open hook at 1731808580940Writing region info on filesystem at 1731808580940Initializing all the Stores at 1731808580942 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808580942Cleaning up temporary data from old regions at 1731808580949 (+7 ms)Running coprocessor post-open hooks at 1731808580956 (+7 ms)Region opened successfully at 1731808580956 2024-11-17T01:56:20,957 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94., pid=15, masterSystemTime=1731808580927 2024-11-17T01:56:20,960 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:56:20,960 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:56:20,961 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=850615d431cafffcd9ff2c59be97ff94, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:20,964 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 850615d431cafffcd9ff2c59be97ff94, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:20,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-17T01:56:20,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 850615d431cafffcd9ff2c59be97ff94, server=c6c0d7a0a7c0,38643,1731808526242 in 194 msec 2024-11-17T01:56:20,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T01:56:20,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=850615d431cafffcd9ff2c59be97ff94, ASSIGN in 359 msec 2024-11-17T01:56:20,974 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:20,975 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808580974"}]},"ts":"1731808580974"} 2024-11-17T01:56:20,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-17T01:56:20,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:20,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2210 sec 2024-11-17T01:56:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T01:56:22,787 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-17T01:56:22,787 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-17T01:56:22,788 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:22,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-17T01:56:22,796 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:22,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-17T01:56:22,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:22,814 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:22,815 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-17T01:56:22,819 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-17T01:56:22,822 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-17T01:56:22,824 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-17T01:56:22,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=207735104112427033 2024-11-17T01:56:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808585824, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:22,830 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-17T01:56:22,831 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-17T01:56:22,832 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:22,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:22,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=207735104112427035 2024-11-17T01:56:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808585833, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:22,837 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:22,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=207735104112427036 2024-11-17T01:56:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808585837, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:22,840 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:22,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=207735104112427037 2024-11-17T01:56:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:52434 deadline: 1731808585840, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-17T01:56:22,844 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-17T01:56:22,851 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1157062217_22 at /127.0.0.1:50380 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=148 (was 161), ProcessCount=11 (was 11), AvailableMemoryMB=6208 (was 6211) 2024-11-17T01:56:22,860 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=148, ProcessCount=11, AvailableMemoryMB=6208 2024-11-17T01:56:22,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-17T01:56:22,866 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:22,866 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:22,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-17T01:56:22,868 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-17T01:56:22,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741843_1019 (size=418) 2024-11-17T01:56:23,286 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e90e10a3b126b34ec9c5c884997a2611, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:23,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741844_1020 (size=69) 2024-11-17T01:56:23,542 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-17T01:56:23,700 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:23,701 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing e90e10a3b126b34ec9c5c884997a2611, disabling compactions & flushes 2024-11-17T01:56:23,701 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:56:23,701 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:56:23,701 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. after waiting 0 ms 2024-11-17T01:56:23,701 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:56:23,702 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:56:23,702 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for e90e10a3b126b34ec9c5c884997a2611: Waiting for close lock at 1731808583701Disabling compacts and flushes for region at 1731808583701Disabling writes for close at 1731808583701Writing region close event to WAL at 1731808583701Closed at 1731808583701 2024-11-17T01:56:23,706 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:23,706 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731808583706"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808583706"}]},"ts":"1731808583706"} 2024-11-17T01:56:23,711 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:23,713 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:23,714 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808583713"}]},"ts":"1731808583713"} 2024-11-17T01:56:23,717 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-17T01:56:23,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e90e10a3b126b34ec9c5c884997a2611, ASSIGN}] 2024-11-17T01:56:23,720 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e90e10a3b126b34ec9c5c884997a2611, ASSIGN 2024-11-17T01:56:23,721 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e90e10a3b126b34ec9c5c884997a2611, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:23,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e90e10a3b126b34ec9c5c884997a2611, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-17T01:56:23,878 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e90e10a3b126b34ec9c5c884997a2611, ASSIGN because future has completed 2024-11-17T01:56:23,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e90e10a3b126b34ec9c5c884997a2611, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:24,046 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:56:24,046 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => e90e10a3b126b34ec9c5c884997a2611, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:24,046 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,046 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:24,047 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,047 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,048 INFO [StoreOpener-e90e10a3b126b34ec9c5c884997a2611-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,050 INFO [StoreOpener-e90e10a3b126b34ec9c5c884997a2611-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e90e10a3b126b34ec9c5c884997a2611 columnFamilyName testFamily 2024-11-17T01:56:24,050 DEBUG [StoreOpener-e90e10a3b126b34ec9c5c884997a2611-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:24,051 INFO [StoreOpener-e90e10a3b126b34ec9c5c884997a2611-1 {}] regionserver.HStore(327): Store=e90e10a3b126b34ec9c5c884997a2611/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:24,051 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,052 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,053 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,053 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,053 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,056 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,059 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:24,060 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened e90e10a3b126b34ec9c5c884997a2611; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73968224, jitterRate=0.10221242904663086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:24,060 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:56:24,061 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for e90e10a3b126b34ec9c5c884997a2611: Running coprocessor pre-open hook at 1731808584047Writing region info on filesystem at 1731808584047Initializing all the Stores at 1731808584048 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808584048Cleaning up temporary data from old regions at 1731808584053 (+5 ms)Running coprocessor post-open hooks at 1731808584060 (+7 ms)Region opened successfully at 1731808584061 (+1 ms) 2024-11-17T01:56:24,063 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., pid=18, masterSystemTime=1731808584034 2024-11-17T01:56:24,066 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:56:24,066 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:56:24,067 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e90e10a3b126b34ec9c5c884997a2611, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:24,070 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e90e10a3b126b34ec9c5c884997a2611, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:24,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T01:56:24,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure e90e10a3b126b34ec9c5c884997a2611, server=c6c0d7a0a7c0,38643,1731808526242 in 193 msec 2024-11-17T01:56:24,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-17T01:56:24,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e90e10a3b126b34ec9c5c884997a2611, ASSIGN in 358 msec 2024-11-17T01:56:24,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:24,080 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808584080"}]},"ts":"1731808584080"} 2024-11-17T01:56:24,082 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-17T01:56:24,084 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:24,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2220 sec 2024-11-17T01:56:24,594 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:56:25,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-17T01:56:25,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:25,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-17T01:56:25,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-17T01:56:25,886 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-17T01:56:25,887 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-17T01:56:25,887 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:25,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-17T01:56:25,893 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:25,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-17T01:56:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:25,907 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:25,907 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-17T01:56:25,910 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-17T01:56:25,912 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-17T01:56:26,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:56:26,915 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=c6c0d7a0a7c0:38643 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-17T01:56:26,915 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-17T01:56:26,916 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 from cache 2024-11-17T01:56:27,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-17T01:56:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:27,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-17T01:56:28,915 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=c6c0d7a0a7c0:38643 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=969ms, rpcTimeout=965ms 2024-11-17T01:56:28,916 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=969ms, rpcTimeout=965ms 2024-11-17T01:56:28,916 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 from cache 2024-11-17T01:56:28,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=136 (was 148), ProcessCount=11 (was 11), AvailableMemoryMB=6196 (was 6208) 2024-11-17T01:56:28,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=136, ProcessCount=11, AvailableMemoryMB=6196 2024-11-17T01:56:28,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-17T01:56:28,942 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:28,943 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:28,943 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-17T01:56:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:56:28,944 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:28,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741845_1021 (size=416) 2024-11-17T01:56:29,354 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7eae297055c9fa234af9f1ed424b842e, NAME => 'testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:29,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741846_1022 (size=67) 2024-11-17T01:56:29,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-17T01:56:29,545 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-17T01:56:29,765 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:29,766 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 7eae297055c9fa234af9f1ed424b842e, disabling compactions & flushes 2024-11-17T01:56:29,766 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:56:29,766 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:56:29,766 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. after waiting 0 ms 2024-11-17T01:56:29,767 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:56:29,767 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:56:29,767 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7eae297055c9fa234af9f1ed424b842e: Waiting for close lock at 1731808589766Disabling compacts and flushes for region at 1731808589766Disabling writes for close at 1731808589766Writing region close event to WAL at 1731808589767 (+1 ms)Closed at 1731808589767 2024-11-17T01:56:29,771 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:29,771 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731808589771"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808589771"}]},"ts":"1731808589771"} 2024-11-17T01:56:29,775 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:29,777 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:29,778 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808589777"}]},"ts":"1731808589777"} 2024-11-17T01:56:29,781 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-17T01:56:29,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7eae297055c9fa234af9f1ed424b842e, ASSIGN}] 2024-11-17T01:56:29,783 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7eae297055c9fa234af9f1ed424b842e, ASSIGN 2024-11-17T01:56:29,785 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7eae297055c9fa234af9f1ed424b842e, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:29,936 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7eae297055c9fa234af9f1ed424b842e, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:29,943 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7eae297055c9fa234af9f1ed424b842e, ASSIGN because future has completed 2024-11-17T01:56:29,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eae297055c9fa234af9f1ed424b842e, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:56:30,104 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:56:30,105 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 7eae297055c9fa234af9f1ed424b842e, NAME => 'testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:30,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:30,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,106 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,108 INFO [StoreOpener-7eae297055c9fa234af9f1ed424b842e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,110 INFO [StoreOpener-7eae297055c9fa234af9f1ed424b842e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eae297055c9fa234af9f1ed424b842e columnFamilyName testFamily 2024-11-17T01:56:30,110 DEBUG [StoreOpener-7eae297055c9fa234af9f1ed424b842e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:30,111 INFO [StoreOpener-7eae297055c9fa234af9f1ed424b842e-1 {}] regionserver.HStore(327): Store=7eae297055c9fa234af9f1ed424b842e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:30,111 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,113 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,113 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,114 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,114 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,117 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,120 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:30,121 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 7eae297055c9fa234af9f1ed424b842e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67992764, jitterRate=0.013171136379241943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:30,121 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:56:30,122 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 7eae297055c9fa234af9f1ed424b842e: Running coprocessor pre-open hook at 1731808590106Writing region info on filesystem at 1731808590106Initializing all the Stores at 1731808590108 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808590108Cleaning up temporary data from old regions at 1731808590114 (+6 ms)Running coprocessor post-open hooks at 1731808590121 (+7 ms)Region opened successfully at 1731808590122 (+1 ms) 2024-11-17T01:56:30,123 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e., pid=21, masterSystemTime=1731808590099 2024-11-17T01:56:30,126 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:56:30,126 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:56:30,127 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7eae297055c9fa234af9f1ed424b842e, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:30,130 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eae297055c9fa234af9f1ed424b842e, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:30,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-17T01:56:30,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 7eae297055c9fa234af9f1ed424b842e, server=c6c0d7a0a7c0,38643,1731808526242 in 187 msec 2024-11-17T01:56:30,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T01:56:30,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7eae297055c9fa234af9f1ed424b842e, ASSIGN in 353 msec 2024-11-17T01:56:30,138 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:30,138 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808590138"}]},"ts":"1731808590138"} 2024-11-17T01:56:30,140 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-17T01:56:30,141 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:30,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2020 sec 2024-11-17T01:56:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T01:56:31,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-17T01:56:31,998 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-17T01:56:31,999 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:32,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-17T01:56:32,010 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:32,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-17T01:56:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:32,025 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:32,025 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-17T01:56:32,028 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-17T01:56:32,030 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-17T01:56:32,031 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-17T01:56:32,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-17T01:56:35,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38643 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427048' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-17T01:56:35,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38643 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:52434 deadline: 1731808596043, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427048' does not exist 2024-11-17T01:56:35,046 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to c6c0d7a0a7c0,38643,1731808526242 for closing scanner id = 207735104112427048 for 7eae297055c9fa234af9f1ed424b842e of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427048' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:56:35,050 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 136), ProcessCount=11 (was 11), AvailableMemoryMB=6195 (was 6196) 2024-11-17T01:56:35,058 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=6195 2024-11-17T01:56:35,060 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-17T01:56:35,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:35,064 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:35,065 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-17T01:56:35,066 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-17T01:56:35,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741847_1023 (size=411) 2024-11-17T01:56:35,483 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fae0a7457f426490220f1a21a6f9018e, NAME => 'testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741848_1024 (size=62) 2024-11-17T01:56:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-17T01:56:35,610 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:56:35,632 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-17T01:56:35,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-17T01:56:35,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:35,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:35,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing fae0a7457f426490220f1a21a6f9018e, disabling compactions & flushes 2024-11-17T01:56:35,898 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:56:35,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:56:35,899 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. after waiting 0 ms 2024-11-17T01:56:35,899 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:56:35,899 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:56:35,899 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for fae0a7457f426490220f1a21a6f9018e: Waiting for close lock at 1731808595898Disabling compacts and flushes for region at 1731808595898Disabling writes for close at 1731808595899 (+1 ms)Writing region close event to WAL at 1731808595899Closed at 1731808595899 2024-11-17T01:56:35,902 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:35,903 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731808595902"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808595902"}]},"ts":"1731808595902"} 2024-11-17T01:56:35,906 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:35,908 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:35,908 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808595908"}]},"ts":"1731808595908"} 2024-11-17T01:56:35,912 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-17T01:56:35,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fae0a7457f426490220f1a21a6f9018e, ASSIGN}] 2024-11-17T01:56:35,914 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fae0a7457f426490220f1a21a6f9018e, ASSIGN 2024-11-17T01:56:35,916 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fae0a7457f426490220f1a21a6f9018e, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:36,067 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=fae0a7457f426490220f1a21a6f9018e, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:36,076 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fae0a7457f426490220f1a21a6f9018e, ASSIGN because future has completed 2024-11-17T01:56:36,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure fae0a7457f426490220f1a21a6f9018e, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-17T01:56:36,242 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:56:36,242 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => fae0a7457f426490220f1a21a6f9018e, NAME => 'testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:36,242 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,242 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:36,243 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,243 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,245 INFO [StoreOpener-fae0a7457f426490220f1a21a6f9018e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,248 INFO [StoreOpener-fae0a7457f426490220f1a21a6f9018e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fae0a7457f426490220f1a21a6f9018e columnFamilyName testFamily 2024-11-17T01:56:36,248 DEBUG [StoreOpener-fae0a7457f426490220f1a21a6f9018e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:36,249 INFO [StoreOpener-fae0a7457f426490220f1a21a6f9018e-1 {}] regionserver.HStore(327): Store=fae0a7457f426490220f1a21a6f9018e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:36,249 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,251 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,251 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,252 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,252 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,254 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,257 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:36,258 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened fae0a7457f426490220f1a21a6f9018e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69120530, jitterRate=0.02997615933418274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:36,258 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:56:36,258 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for fae0a7457f426490220f1a21a6f9018e: Running coprocessor pre-open hook at 1731808596243Writing region info on filesystem at 1731808596243Initializing all the Stores at 1731808596245 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808596245Cleaning up temporary data from old regions at 1731808596252 (+7 ms)Running coprocessor post-open hooks at 1731808596258 (+6 ms)Region opened successfully at 1731808596258 2024-11-17T01:56:36,260 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e., pid=24, masterSystemTime=1731808596231 2024-11-17T01:56:36,262 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:56:36,262 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:56:36,263 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=fae0a7457f426490220f1a21a6f9018e, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:36,265 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure fae0a7457f426490220f1a21a6f9018e, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:36,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T01:56:36,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure fae0a7457f426490220f1a21a6f9018e, server=c6c0d7a0a7c0,38643,1731808526242 in 191 msec 2024-11-17T01:56:36,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-17T01:56:36,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=fae0a7457f426490220f1a21a6f9018e, ASSIGN in 357 msec 2024-11-17T01:56:36,274 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:36,274 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808596274"}]},"ts":"1731808596274"} 2024-11-17T01:56:36,277 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-17T01:56:36,278 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:36,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2170 sec 2024-11-17T01:56:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-17T01:56:38,118 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-17T01:56:38,118 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-17T01:56:38,119 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:38,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-17T01:56:38,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:38,131 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-17T01:56:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:38,145 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:38,145 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-17T01:56:38,149 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-17T01:56:38,152 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-17T01:56:38,154 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-17T01:56:38,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-17T01:56:40,937 INFO [master/c6c0d7a0a7c0:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T01:56:40,938 INFO [master/c6c0d7a0a7c0:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T01:56:41,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38643 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427052' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-17T01:56:41,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:52434 deadline: 1731808602162, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427052' does not exist 2024-11-17T01:56:41,165 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to c6c0d7a0a7c0,38643,1731808526242 for closing scanner id = 207735104112427052 for fae0a7457f426490220f1a21a6f9018e of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427052' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:56:41,172 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=123 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=6191 (was 6195) 2024-11-17T01:56:41,180 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=123, ProcessCount=11, AvailableMemoryMB=6191 2024-11-17T01:56:41,183 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:41,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-17T01:56:41,186 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:41,186 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:41,186 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-17T01:56:41,188 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T01:56:41,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741849_1025 (size=419) 2024-11-17T01:56:41,603 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => db7dff0f0deacc57eb5f521f63058137, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:41,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741850_1026 (size=70) 2024-11-17T01:56:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-17T01:56:41,745 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:56:41,790 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-17T01:56:42,016 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:42,017 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing db7dff0f0deacc57eb5f521f63058137, disabling compactions & flushes 2024-11-17T01:56:42,017 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:56:42,017 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:56:42,017 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. after waiting 0 ms 2024-11-17T01:56:42,017 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:56:42,017 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:56:42,017 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for db7dff0f0deacc57eb5f521f63058137: Waiting for close lock at 1731808602017Disabling compacts and flushes for region at 1731808602017Disabling writes for close at 1731808602017Writing region close event to WAL at 1731808602017Closed at 1731808602017 2024-11-17T01:56:42,019 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:42,019 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731808602019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808602019"}]},"ts":"1731808602019"} 2024-11-17T01:56:42,022 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:42,023 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:42,023 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808602023"}]},"ts":"1731808602023"} 2024-11-17T01:56:42,026 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-17T01:56:42,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=db7dff0f0deacc57eb5f521f63058137, ASSIGN}] 2024-11-17T01:56:42,028 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=db7dff0f0deacc57eb5f521f63058137, ASSIGN 2024-11-17T01:56:42,029 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=db7dff0f0deacc57eb5f521f63058137, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:42,180 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=db7dff0f0deacc57eb5f521f63058137, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:42,185 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=db7dff0f0deacc57eb5f521f63058137, ASSIGN because future has completed 2024-11-17T01:56:42,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure db7dff0f0deacc57eb5f521f63058137, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T01:56:42,348 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:56:42,348 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => db7dff0f0deacc57eb5f521f63058137, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:42,349 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,349 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:42,349 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,349 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,352 INFO [StoreOpener-db7dff0f0deacc57eb5f521f63058137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,355 INFO [StoreOpener-db7dff0f0deacc57eb5f521f63058137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db7dff0f0deacc57eb5f521f63058137 columnFamilyName testFamily 2024-11-17T01:56:42,355 DEBUG [StoreOpener-db7dff0f0deacc57eb5f521f63058137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:42,356 INFO [StoreOpener-db7dff0f0deacc57eb5f521f63058137-1 {}] regionserver.HStore(327): Store=db7dff0f0deacc57eb5f521f63058137/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:42,356 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,358 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,360 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,360 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,363 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:42,367 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened db7dff0f0deacc57eb5f521f63058137; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65758598, jitterRate=-0.0201205313205719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:42,367 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:56:42,368 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for db7dff0f0deacc57eb5f521f63058137: Running coprocessor pre-open hook at 1731808602349Writing region info on filesystem at 1731808602349Initializing all the Stores at 1731808602351 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808602351Cleaning up temporary data from old regions at 1731808602360 (+9 ms)Running coprocessor post-open hooks at 1731808602367 (+7 ms)Region opened successfully at 1731808602368 (+1 ms) 2024-11-17T01:56:42,369 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., pid=27, masterSystemTime=1731808602342 2024-11-17T01:56:42,371 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:56:42,372 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:56:42,372 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=db7dff0f0deacc57eb5f521f63058137, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:42,374 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure db7dff0f0deacc57eb5f521f63058137, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:42,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-17T01:56:42,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure db7dff0f0deacc57eb5f521f63058137, server=c6c0d7a0a7c0,38643,1731808526242 in 189 msec 2024-11-17T01:56:42,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T01:56:42,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=db7dff0f0deacc57eb5f521f63058137, ASSIGN in 352 msec 2024-11-17T01:56:42,381 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:42,381 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808602381"}]},"ts":"1731808602381"} 2024-11-17T01:56:42,383 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-17T01:56:42,384 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:42,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2010 sec 2024-11-17T01:56:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T01:56:44,206 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-17T01:56:44,206 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-17T01:56:44,207 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:44,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-17T01:56:44,217 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:44,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-17T01:56:44,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:44,228 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:44,228 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-17T01:56:44,231 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-17T01:56:44,232 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-17T01:56:45,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-17T01:56:45,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:45,876 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-17T01:56:45,876 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:47,855 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:56:47,892 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-17T01:56:50,235 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=c6c0d7a0a7c0:38643 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-17T01:56:50,235 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-17T01:56:50,235 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 from cache 2024-11-17T01:56:50,252 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=104 (was 123), ProcessCount=11 (was 11), AvailableMemoryMB=6186 (was 6191) 2024-11-17T01:56:50,262 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=104, ProcessCount=11, AvailableMemoryMB=6186 2024-11-17T01:56:50,264 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:50,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-17T01:56:50,267 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:50,267 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:50,267 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-17T01:56:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-17T01:56:50,269 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:50,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741851_1027 (size=407) 2024-11-17T01:56:50,684 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a04bf5d01b7b82d0981ed564d7e5fe29, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:50,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741852_1028 (size=58) 2024-11-17T01:56:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-17T01:56:51,100 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:51,100 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing a04bf5d01b7b82d0981ed564d7e5fe29, disabling compactions & flushes 2024-11-17T01:56:51,101 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:56:51,101 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:56:51,101 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. after waiting 0 ms 2024-11-17T01:56:51,101 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:56:51,101 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:56:51,102 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for a04bf5d01b7b82d0981ed564d7e5fe29: Waiting for close lock at 1731808611100Disabling compacts and flushes for region at 1731808611100Disabling writes for close at 1731808611101 (+1 ms)Writing region close event to WAL at 1731808611101Closed at 1731808611101 2024-11-17T01:56:51,106 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:56:51,107 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731808611107"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808611107"}]},"ts":"1731808611107"} 2024-11-17T01:56:51,111 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:56:51,112 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:56:51,113 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808611112"}]},"ts":"1731808611112"} 2024-11-17T01:56:51,116 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-17T01:56:51,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a04bf5d01b7b82d0981ed564d7e5fe29, ASSIGN}] 2024-11-17T01:56:51,118 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a04bf5d01b7b82d0981ed564d7e5fe29, ASSIGN 2024-11-17T01:56:51,119 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a04bf5d01b7b82d0981ed564d7e5fe29, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:56:51,271 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a04bf5d01b7b82d0981ed564d7e5fe29, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:51,277 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a04bf5d01b7b82d0981ed564d7e5fe29, ASSIGN because future has completed 2024-11-17T01:56:51,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a04bf5d01b7b82d0981ed564d7e5fe29, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:56:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-17T01:56:51,445 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:56:51,445 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => a04bf5d01b7b82d0981ed564d7e5fe29, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:56:51,445 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,446 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:56:51,446 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,446 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,448 INFO [StoreOpener-a04bf5d01b7b82d0981ed564d7e5fe29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,450 INFO [StoreOpener-a04bf5d01b7b82d0981ed564d7e5fe29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a04bf5d01b7b82d0981ed564d7e5fe29 columnFamilyName testFamily 2024-11-17T01:56:51,450 DEBUG [StoreOpener-a04bf5d01b7b82d0981ed564d7e5fe29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:51,451 INFO [StoreOpener-a04bf5d01b7b82d0981ed564d7e5fe29-1 {}] regionserver.HStore(327): Store=a04bf5d01b7b82d0981ed564d7e5fe29/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:56:51,451 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,453 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,454 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,454 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,454 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,457 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,460 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:56:51,461 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened a04bf5d01b7b82d0981ed564d7e5fe29; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60104188, jitterRate=-0.10437780618667603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:56:51,461 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:56:51,462 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for a04bf5d01b7b82d0981ed564d7e5fe29: Running coprocessor pre-open hook at 1731808611446Writing region info on filesystem at 1731808611446Initializing all the Stores at 1731808611447 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808611448 (+1 ms)Cleaning up temporary data from old regions at 1731808611454 (+6 ms)Running coprocessor post-open hooks at 1731808611461 (+7 ms)Region opened successfully at 1731808611462 (+1 ms) 2024-11-17T01:56:51,463 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29., pid=30, masterSystemTime=1731808611433 2024-11-17T01:56:51,465 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:56:51,465 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:56:51,466 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a04bf5d01b7b82d0981ed564d7e5fe29, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:56:51,468 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a04bf5d01b7b82d0981ed564d7e5fe29, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:56:51,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T01:56:51,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure a04bf5d01b7b82d0981ed564d7e5fe29, server=c6c0d7a0a7c0,38643,1731808526242 in 192 msec 2024-11-17T01:56:51,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-17T01:56:51,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a04bf5d01b7b82d0981ed564d7e5fe29, ASSIGN in 356 msec 2024-11-17T01:56:51,477 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:56:51,477 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808611477"}]},"ts":"1731808611477"} 2024-11-17T01:56:51,479 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-17T01:56:51,481 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:56:51,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2170 sec 2024-11-17T01:56:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-17T01:56:53,308 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-17T01:56:53,308 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-17T01:56:53,309 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:53,317 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-17T01:56:53,318 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:56:53,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-17T01:56:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:56:53,332 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:56:53,332 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-17T01:56:53,335 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-17T01:56:53,336 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-17T01:56:53,337 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-17T01:56:53,338 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-17T01:56:53,895 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-17T01:56:54,594 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T01:56:55,874 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-17T01:56:55,874 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-17T01:56:56,950 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:56:59,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38643 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427060' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-17T01:56:59,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38643 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:52434 deadline: 1731808625346, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427060' does not exist 2024-11-17T01:56:59,349 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to c6c0d7a0a7c0,38643,1731808526242 for closing scanner id = 207735104112427060 for a04bf5d01b7b82d0981ed564d7e5fe29 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '207735104112427060' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T01:56:59,356 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=96 (was 104), ProcessCount=11 (was 11), AvailableMemoryMB=6170 (was 6186) 2024-11-17T01:56:59,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=96, ProcessCount=11, AvailableMemoryMB=6170 2024-11-17T01:56:59,366 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T01:56:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-17T01:56:59,369 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T01:56:59,369 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:56:59,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-17T01:56:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T01:56:59,370 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T01:56:59,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741853_1029 (size=423) 2024-11-17T01:56:59,782 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4bf2d331570c103643c58e369af12815, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067 2024-11-17T01:56:59,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741854_1030 (size=74) 2024-11-17T01:56:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-17T01:57:00,179 DEBUG [master/c6c0d7a0a7c0:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-17T01:57:00,192 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:57:00,192 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 4bf2d331570c103643c58e369af12815, disabling compactions & flushes 2024-11-17T01:57:00,192 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:00,192 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:00,192 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. after waiting 0 ms 2024-11-17T01:57:00,192 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:00,192 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:00,192 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4bf2d331570c103643c58e369af12815: Waiting for close lock at 1731808620192Disabling compacts and flushes for region at 1731808620192Disabling writes for close at 1731808620192Writing region close event to WAL at 1731808620192Closed at 1731808620192 2024-11-17T01:57:00,194 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T01:57:00,194 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731808620194"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731808620194"}]},"ts":"1731808620194"} 2024-11-17T01:57:00,197 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T01:57:00,198 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T01:57:00,198 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808620198"}]},"ts":"1731808620198"} 2024-11-17T01:57:00,200 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-17T01:57:00,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=4bf2d331570c103643c58e369af12815, ASSIGN}] 2024-11-17T01:57:00,202 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=4bf2d331570c103643c58e369af12815, ASSIGN 2024-11-17T01:57:00,203 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=4bf2d331570c103643c58e369af12815, ASSIGN; state=OFFLINE, location=c6c0d7a0a7c0,38643,1731808526242; forceNewPlan=false, retain=false 2024-11-17T01:57:00,354 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=4bf2d331570c103643c58e369af12815, regionState=OPENING, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:57:00,359 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=4bf2d331570c103643c58e369af12815, ASSIGN because future has completed 2024-11-17T01:57:00,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4bf2d331570c103643c58e369af12815, server=c6c0d7a0a7c0,38643,1731808526242}] 2024-11-17T01:57:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T01:57:00,527 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:00,527 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 4bf2d331570c103643c58e369af12815, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815.', STARTKEY => '', ENDKEY => ''} 2024-11-17T01:57:00,528 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,528 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T01:57:00,528 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,528 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,531 INFO [StoreOpener-4bf2d331570c103643c58e369af12815-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,533 INFO [StoreOpener-4bf2d331570c103643c58e369af12815-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4bf2d331570c103643c58e369af12815 columnFamilyName testFamily 2024-11-17T01:57:00,533 DEBUG [StoreOpener-4bf2d331570c103643c58e369af12815-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T01:57:00,534 INFO [StoreOpener-4bf2d331570c103643c58e369af12815-1 {}] regionserver.HStore(327): Store=4bf2d331570c103643c58e369af12815/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T01:57:00,534 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,536 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,536 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,537 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,537 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,540 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,542 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T01:57:00,543 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 4bf2d331570c103643c58e369af12815; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71338612, jitterRate=0.06302815675735474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T01:57:00,543 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:00,544 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 4bf2d331570c103643c58e369af12815: Running coprocessor pre-open hook at 1731808620528Writing region info on filesystem at 1731808620528Initializing all the Stores at 1731808620530 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731808620530Cleaning up temporary data from old regions at 1731808620537 (+7 ms)Running coprocessor post-open hooks at 1731808620543 (+6 ms)Region opened successfully at 1731808620544 (+1 ms) 2024-11-17T01:57:00,545 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., pid=33, masterSystemTime=1731808620513 2024-11-17T01:57:00,547 DEBUG [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:00,547 INFO [RS_OPEN_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:00,548 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=4bf2d331570c103643c58e369af12815, regionState=OPEN, openSeqNum=2, regionLocation=c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:57:00,550 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4bf2d331570c103643c58e369af12815, server=c6c0d7a0a7c0,38643,1731808526242 because future has completed 2024-11-17T01:57:00,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-17T01:57:00,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 4bf2d331570c103643c58e369af12815, server=c6c0d7a0a7c0,38643,1731808526242 in 193 msec 2024-11-17T01:57:00,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T01:57:00,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=4bf2d331570c103643c58e369af12815, ASSIGN in 353 msec 2024-11-17T01:57:00,558 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T01:57:00,558 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731808620558"}]},"ts":"1731808620558"} 2024-11-17T01:57:00,560 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-17T01:57:00,561 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T01:57:00,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1950 sec 2024-11-17T01:57:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33177 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-17T01:57:02,416 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-17T01:57:02,417 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-17T01:57:02,417 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:57:02,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-17T01:57:02,427 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T01:57:02,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-17T01:57:02,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:57:02,440 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-17T01:57:02,440 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-17T01:57:02,444 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-17T01:57:02,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-17T01:57:03,455 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=c6c0d7a0a7c0:38643 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-17T01:57:03,455 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-17T01:57:03,455 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 from cache 2024-11-17T01:57:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-17T01:57:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2] 2024-11-17T01:57:04,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38643 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-17T01:57:05,455 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=c6c0d7a0a7c0:38643 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=974ms 2024-11-17T01:57:05,455 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=974ms 2024-11-17T01:57:05,456 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., hostname=c6c0d7a0a7c0,38643,1731808526242, seqNum=2 from cache 2024-11-17T01:57:05,469 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=88 (was 96), ProcessCount=11 (was 11), AvailableMemoryMB=6166 (was 6170) 2024-11-17T01:57:05,470 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:57:05,470 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:57:05,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:57:05,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:57:05,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:57:05,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T01:57:05,478 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:57:05,478 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:57:05,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:57:05,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:57:05,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T01:57:05,478 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T01:57:05,479 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1279614773, stopped=false 2024-11-17T01:57:05,479 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c6c0d7a0a7c0,33177,1731808525509 2024-11-17T01:57:05,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:57:05,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T01:57:05,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:57:05,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:57:05,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:57:05,531 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T01:57:05,531 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:57:05,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:57:05,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:57:05,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T01:57:05,532 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6c0d7a0a7c0,38643,1731808526242' ***** 2024-11-17T01:57:05,532 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T01:57:05,533 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T01:57:05,534 INFO [RS:0;c6c0d7a0a7c0:38643 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T01:57:05,534 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T01:57:05,534 INFO [RS:0;c6c0d7a0a7c0:38643 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T01:57:05,534 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ac2a5bf7f225df52619c184f8fa701c 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for db7dff0f0deacc57eb5f521f63058137 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for 850615d431cafffcd9ff2c59be97ff94 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for a04bf5d01b7b82d0981ed564d7e5fe29 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for 4bf2d331570c103643c58e369af12815 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for 1f7df8e6f82328512994f55ec7a732b7 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:05,535 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ac2a5bf7f225df52619c184f8fa701c, disabling compactions & flushes 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for 7eae297055c9fa234af9f1ed424b842e 2024-11-17T01:57:05,535 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:57:05,535 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:57:05,535 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:57:05,536 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. after waiting 0 ms 2024-11-17T01:57:05,536 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(959): stopping server c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:57:05,536 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:57:05,536 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:57:05,536 INFO [RS:0;c6c0d7a0a7c0:38643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c6c0d7a0a7c0:38643. 2024-11-17T01:57:05,536 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T01:57:05,536 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:57:05,536 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T01:57:05,536 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T01:57:05,536 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T01:57:05,537 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T01:57:05,537 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-17T01:57:05,537 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T01:57:05,537 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T01:57:05,537 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T01:57:05,537 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T01:57:05,537 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T01:57:05,537 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8ac2a5bf7f225df52619c184f8fa701c, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, db7dff0f0deacc57eb5f521f63058137, e90e10a3b126b34ec9c5c884997a2611, fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:57:05,539 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-17T01:57:05,539 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8ac2a5bf7f225df52619c184f8fa701c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:05,595 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c/.tmp/testFamily/43b470e0dc554e58baf44299489b7575 is 61, key is row-0/testFamily:testQualifier/1731808573713/Put/seqid=0 2024-11-17T01:57:05,600 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/info/78a6411c52a84ea19719053aa6c8dc21 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7./info:regioninfo/1731808561230/Put/seqid=0 2024-11-17T01:57:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741855_1031 (size=5049) 2024-11-17T01:57:05,604 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c/.tmp/testFamily/43b470e0dc554e58baf44299489b7575 2024-11-17T01:57:05,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741856_1032 (size=20564) 2024-11-17T01:57:05,644 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c/.tmp/testFamily/43b470e0dc554e58baf44299489b7575 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c/testFamily/43b470e0dc554e58baf44299489b7575 2024-11-17T01:57:05,653 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c/testFamily/43b470e0dc554e58baf44299489b7575, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:05,661 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8ac2a5bf7f225df52619c184f8fa701c in 121ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:05,668 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNextAsync/8ac2a5bf7f225df52619c184f8fa701c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:05,670 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:57:05,670 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ac2a5bf7f225df52619c184f8fa701c: Waiting for close lock at 1731808625535Running coprocessor pre-close hooks at 1731808625535Disabling compacts and flushes for region at 1731808625535Disabling writes for close at 1731808625536 (+1 ms)Obtaining lock to block concurrent updates at 1731808625539 (+3 ms)Preparing flush snapshotting stores in 8ac2a5bf7f225df52619c184f8fa701c at 1731808625539Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808625546 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. at 1731808625547 (+1 ms)Flushing 8ac2a5bf7f225df52619c184f8fa701c/testFamily: creating writer at 1731808625549 (+2 ms)Flushing 8ac2a5bf7f225df52619c184f8fa701c/testFamily: appending metadata at 1731808625590 (+41 ms)Flushing 8ac2a5bf7f225df52619c184f8fa701c/testFamily: closing flushed file at 1731808625592 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@707a55ec: reopening flushed file at 1731808625642 (+50 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8ac2a5bf7f225df52619c184f8fa701c in 121ms, sequenceid=8, compaction requested=false at 1731808625661 (+19 ms)Writing region close event to WAL at 1731808625663 (+2 ms)Running coprocessor post-close hooks at 1731808625669 (+6 ms)Closed at 1731808625670 (+1 ms) 2024-11-17T01:57:05,671 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731808570671.8ac2a5bf7f225df52619c184f8fa701c. 2024-11-17T01:57:05,671 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fae0a7457f426490220f1a21a6f9018e, disabling compactions & flushes 2024-11-17T01:57:05,671 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:57:05,671 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:57:05,671 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. after waiting 0 ms 2024-11-17T01:57:05,671 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:57:05,671 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fae0a7457f426490220f1a21a6f9018e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:05,688 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e/.tmp/testFamily/ccd6e365bda24e309322e98f82e94be8 is 61, key is row-0/testFamily:testQualifier/1731808598135/Put/seqid=0 2024-11-17T01:57:05,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741857_1033 (size=5049) 2024-11-17T01:57:05,737 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, db7dff0f0deacc57eb5f521f63058137, e90e10a3b126b34ec9c5c884997a2611, fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:57:05,938 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, db7dff0f0deacc57eb5f521f63058137, e90e10a3b126b34ec9c5c884997a2611, fae0a7457f426490220f1a21a6f9018e 2024-11-17T01:57:05,975 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-17T01:57:06,008 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/info/78a6411c52a84ea19719053aa6c8dc21 2024-11-17T01:57:06,030 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:57:06,041 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/ns/4215ec47431b414fa2c67cd2d1109527 is 43, key is default/ns:d/1731808560125/Put/seqid=0 2024-11-17T01:57:06,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741858_1034 (size=5153) 2024-11-17T01:57:06,047 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/ns/4215ec47431b414fa2c67cd2d1109527 2024-11-17T01:57:06,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-17T01:57:06,066 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-17T01:57:06,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-17T01:57:06,068 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-17T01:57:06,072 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/table/32574a4a434d4888a59d2d14450feb31 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731808561245/Put/seqid=0 2024-11-17T01:57:06,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741859_1035 (size=6699) 2024-11-17T01:57:06,094 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e/.tmp/testFamily/ccd6e365bda24e309322e98f82e94be8 2024-11-17T01:57:06,103 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e/.tmp/testFamily/ccd6e365bda24e309322e98f82e94be8 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e/testFamily/ccd6e365bda24e309322e98f82e94be8 2024-11-17T01:57:06,111 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e/testFamily/ccd6e365bda24e309322e98f82e94be8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:06,112 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fae0a7457f426490220f1a21a6f9018e in 441ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:06,118 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNext/fae0a7457f426490220f1a21a6f9018e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:06,118 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:57:06,119 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fae0a7457f426490220f1a21a6f9018e: Waiting for close lock at 1731808625671Running coprocessor pre-close hooks at 1731808625671Disabling compacts and flushes for region at 1731808625671Disabling writes for close at 1731808625671Obtaining lock to block concurrent updates at 1731808625671Preparing flush snapshotting stores in fae0a7457f426490220f1a21a6f9018e at 1731808625671Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808625671Flushing stores of testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. at 1731808625672 (+1 ms)Flushing fae0a7457f426490220f1a21a6f9018e/testFamily: creating writer at 1731808625672Flushing fae0a7457f426490220f1a21a6f9018e/testFamily: appending metadata at 1731808625687 (+15 ms)Flushing fae0a7457f426490220f1a21a6f9018e/testFamily: closing flushed file at 1731808625688 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62a810a2: reopening flushed file at 1731808626102 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fae0a7457f426490220f1a21a6f9018e in 441ms, sequenceid=8, compaction requested=false at 1731808626113 (+11 ms)Writing region close event to WAL at 1731808626113Running coprocessor post-close hooks at 1731808626118 (+5 ms)Closed at 1731808626118 2024-11-17T01:57:06,119 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731808595060.fae0a7457f426490220f1a21a6f9018e. 2024-11-17T01:57:06,119 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db7dff0f0deacc57eb5f521f63058137, disabling compactions & flushes 2024-11-17T01:57:06,119 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:57:06,119 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:57:06,119 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. after waiting 0 ms 2024-11-17T01:57:06,119 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:57:06,119 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing db7dff0f0deacc57eb5f521f63058137 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:06,138 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, db7dff0f0deacc57eb5f521f63058137, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:06,143 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137/.tmp/testFamily/353a5376816b413db73879c7979079cb is 61, key is row-0/testFamily:testQualifier/1731808604220/Put/seqid=0 2024-11-17T01:57:06,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741860_1036 (size=5049) 2024-11-17T01:57:06,338 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, db7dff0f0deacc57eb5f521f63058137, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:06,479 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/table/32574a4a434d4888a59d2d14450feb31 2024-11-17T01:57:06,492 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/info/78a6411c52a84ea19719053aa6c8dc21 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/info/78a6411c52a84ea19719053aa6c8dc21 2024-11-17T01:57:06,500 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/info/78a6411c52a84ea19719053aa6c8dc21, entries=100, sequenceid=56, filesize=20.1 K 2024-11-17T01:57:06,502 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/ns/4215ec47431b414fa2c67cd2d1109527 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/ns/4215ec47431b414fa2c67cd2d1109527 2024-11-17T01:57:06,507 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T01:57:06,507 INFO [regionserver/c6c0d7a0a7c0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T01:57:06,511 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/ns/4215ec47431b414fa2c67cd2d1109527, entries=2, sequenceid=56, filesize=5.0 K 2024-11-17T01:57:06,512 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/.tmp/table/32574a4a434d4888a59d2d14450feb31 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/table/32574a4a434d4888a59d2d14450feb31 2024-11-17T01:57:06,521 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/table/32574a4a434d4888a59d2d14450feb31, entries=20, sequenceid=56, filesize=6.5 K 2024-11-17T01:57:06,522 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 985ms, sequenceid=56, compaction requested=false 2024-11-17T01:57:06,527 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-17T01:57:06,528 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T01:57:06,528 INFO [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T01:57:06,529 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731808625537Running coprocessor pre-close hooks at 1731808625537Disabling compacts and flushes for region at 1731808625537Disabling writes for close at 1731808625537Obtaining lock to block concurrent updates at 1731808625539 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1731808625539Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731808625546 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731808625547 (+1 ms)Flushing 1588230740/info: creating writer at 1731808625549 (+2 ms)Flushing 1588230740/info: appending metadata at 1731808625595 (+46 ms)Flushing 1588230740/info: closing flushed file at 1731808625595Flushing 1588230740/ns: creating writer at 1731808626023 (+428 ms)Flushing 1588230740/ns: appending metadata at 1731808626040 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731808626040Flushing 1588230740/table: creating writer at 1731808626055 (+15 ms)Flushing 1588230740/table: appending metadata at 1731808626071 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731808626071Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bd6c393: reopening flushed file at 1731808626491 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@147a6a6b: reopening flushed file at 1731808626501 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f429f16: reopening flushed file at 1731808626511 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 985ms, sequenceid=56, compaction requested=false at 1731808626522 (+11 ms)Writing region close event to WAL at 1731808626524 (+2 ms)Running coprocessor post-close hooks at 1731808626528 (+4 ms)Closed at 1731808626528 2024-11-17T01:57:06,529 DEBUG [RS_CLOSE_META-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T01:57:06,539 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1321): Waiting on 8 regions to close 2024-11-17T01:57:06,539 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1325): Online Regions={db7dff0f0deacc57eb5f521f63058137=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., 850615d431cafffcd9ff2c59be97ff94=testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94., a04bf5d01b7b82d0981ed564d7e5fe29=hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29., 4bf2d331570c103643c58e369af12815=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., 1f7df8e6f82328512994f55ec7a732b7=testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7., e90e10a3b126b34ec9c5c884997a2611=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., 7eae297055c9fa234af9f1ed424b842e=testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e., 8f7c51a52b68bf2ed5bf01e5bd055060=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.} 2024-11-17T01:57:06,539 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, db7dff0f0deacc57eb5f521f63058137, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:06,549 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137/.tmp/testFamily/353a5376816b413db73879c7979079cb 2024-11-17T01:57:06,559 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137/.tmp/testFamily/353a5376816b413db73879c7979079cb as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137/testFamily/353a5376816b413db73879c7979079cb 2024-11-17T01:57:06,570 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137/testFamily/353a5376816b413db73879c7979079cb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:06,571 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for db7dff0f0deacc57eb5f521f63058137 in 452ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:06,577 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/db7dff0f0deacc57eb5f521f63058137/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:06,578 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:57:06,578 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db7dff0f0deacc57eb5f521f63058137: Waiting for close lock at 1731808626119Running coprocessor pre-close hooks at 1731808626119Disabling compacts and flushes for region at 1731808626119Disabling writes for close at 1731808626119Obtaining lock to block concurrent updates at 1731808626119Preparing flush snapshotting stores in db7dff0f0deacc57eb5f521f63058137 at 1731808626119Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808626120 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. at 1731808626120Flushing db7dff0f0deacc57eb5f521f63058137/testFamily: creating writer at 1731808626120Flushing db7dff0f0deacc57eb5f521f63058137/testFamily: appending metadata at 1731808626142 (+22 ms)Flushing db7dff0f0deacc57eb5f521f63058137/testFamily: closing flushed file at 1731808626142Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@150e677f: reopening flushed file at 1731808626558 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for db7dff0f0deacc57eb5f521f63058137 in 452ms, sequenceid=8, compaction requested=false at 1731808626571 (+13 ms)Writing region close event to WAL at 1731808626572 (+1 ms)Running coprocessor post-close hooks at 1731808626578 (+6 ms)Closed at 1731808626578 2024-11-17T01:57:06,578 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731808601183.db7dff0f0deacc57eb5f521f63058137. 2024-11-17T01:57:06,579 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 850615d431cafffcd9ff2c59be97ff94, disabling compactions & flushes 2024-11-17T01:57:06,579 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:57:06,579 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:57:06,579 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. after waiting 0 ms 2024-11-17T01:57:06,579 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:57:06,579 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 850615d431cafffcd9ff2c59be97ff94 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:06,594 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94/.tmp/testFamily/bba44ee330a94199a6fd9519ca569a90 is 61, key is row-0/testFamily:testQualifier/1731808582802/Put/seqid=0 2024-11-17T01:57:06,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741861_1037 (size=5049) 2024-11-17T01:57:06,739 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:06,940 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 850615d431cafffcd9ff2c59be97ff94, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:07,002 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94/.tmp/testFamily/bba44ee330a94199a6fd9519ca569a90 2024-11-17T01:57:07,016 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94/.tmp/testFamily/bba44ee330a94199a6fd9519ca569a90 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94/testFamily/bba44ee330a94199a6fd9519ca569a90 2024-11-17T01:57:07,024 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94/testFamily/bba44ee330a94199a6fd9519ca569a90, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:07,025 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 850615d431cafffcd9ff2c59be97ff94 in 446ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:07,031 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextException/850615d431cafffcd9ff2c59be97ff94/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:07,031 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:57:07,032 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 850615d431cafffcd9ff2c59be97ff94: Waiting for close lock at 1731808626579Running coprocessor pre-close hooks at 1731808626579Disabling compacts and flushes for region at 1731808626579Disabling writes for close at 1731808626579Obtaining lock to block concurrent updates at 1731808626579Preparing flush snapshotting stores in 850615d431cafffcd9ff2c59be97ff94 at 1731808626579Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808626579Flushing stores of testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. at 1731808626580 (+1 ms)Flushing 850615d431cafffcd9ff2c59be97ff94/testFamily: creating writer at 1731808626580Flushing 850615d431cafffcd9ff2c59be97ff94/testFamily: appending metadata at 1731808626594 (+14 ms)Flushing 850615d431cafffcd9ff2c59be97ff94/testFamily: closing flushed file at 1731808626594Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5226907b: reopening flushed file at 1731808627015 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 850615d431cafffcd9ff2c59be97ff94 in 446ms, sequenceid=8, compaction requested=false at 1731808627025 (+10 ms)Writing region close event to WAL at 1731808627026 (+1 ms)Running coprocessor post-close hooks at 1731808627031 (+5 ms)Closed at 1731808627031 2024-11-17T01:57:07,032 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731808579758.850615d431cafffcd9ff2c59be97ff94. 2024-11-17T01:57:07,032 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a04bf5d01b7b82d0981ed564d7e5fe29, disabling compactions & flushes 2024-11-17T01:57:07,032 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:57:07,032 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:57:07,032 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. after waiting 0 ms 2024-11-17T01:57:07,032 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:57:07,032 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a04bf5d01b7b82d0981ed564d7e5fe29 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:07,049 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29/.tmp/testFamily/36dadf7fcfc1458c9c43dfe9668ac480 is 61, key is row-0/testFamily:testQualifier/1731808613323/Put/seqid=0 2024-11-17T01:57:07,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741862_1038 (size=5049) 2024-11-17T01:57:07,140 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:07,341 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, a04bf5d01b7b82d0981ed564d7e5fe29, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:07,456 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29/.tmp/testFamily/36dadf7fcfc1458c9c43dfe9668ac480 2024-11-17T01:57:07,473 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29/.tmp/testFamily/36dadf7fcfc1458c9c43dfe9668ac480 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29/testFamily/36dadf7fcfc1458c9c43dfe9668ac480 2024-11-17T01:57:07,481 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29/testFamily/36dadf7fcfc1458c9c43dfe9668ac480, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:07,482 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a04bf5d01b7b82d0981ed564d7e5fe29 in 450ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:07,486 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnNext/a04bf5d01b7b82d0981ed564d7e5fe29/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:07,487 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:57:07,487 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a04bf5d01b7b82d0981ed564d7e5fe29: Waiting for close lock at 1731808627032Running coprocessor pre-close hooks at 1731808627032Disabling compacts and flushes for region at 1731808627032Disabling writes for close at 1731808627032Obtaining lock to block concurrent updates at 1731808627032Preparing flush snapshotting stores in a04bf5d01b7b82d0981ed564d7e5fe29 at 1731808627032Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808627033 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. at 1731808627033Flushing a04bf5d01b7b82d0981ed564d7e5fe29/testFamily: creating writer at 1731808627033Flushing a04bf5d01b7b82d0981ed564d7e5fe29/testFamily: appending metadata at 1731808627049 (+16 ms)Flushing a04bf5d01b7b82d0981ed564d7e5fe29/testFamily: closing flushed file at 1731808627049Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f0c6048: reopening flushed file at 1731808627472 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a04bf5d01b7b82d0981ed564d7e5fe29 in 450ms, sequenceid=8, compaction requested=false at 1731808627482 (+10 ms)Writing region close event to WAL at 1731808627482Running coprocessor post-close hooks at 1731808627487 (+5 ms)Closed at 1731808627487 2024-11-17T01:57:07,487 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731808610264.a04bf5d01b7b82d0981ed564d7e5fe29. 2024-11-17T01:57:07,487 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4bf2d331570c103643c58e369af12815, disabling compactions & flushes 2024-11-17T01:57:07,488 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:07,488 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:07,488 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. after waiting 0 ms 2024-11-17T01:57:07,488 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:07,488 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4bf2d331570c103643c58e369af12815 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:07,503 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815/.tmp/testFamily/d0147f7200764a97bad6764dbe9a87cc is 61, key is row-0/testFamily:testQualifier/1731808622431/Put/seqid=0 2024-11-17T01:57:07,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741863_1039 (size=5049) 2024-11-17T01:57:07,541 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-17T01:57:07,541 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1325): Online Regions={4bf2d331570c103643c58e369af12815=testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., 1f7df8e6f82328512994f55ec7a732b7=testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7., e90e10a3b126b34ec9c5c884997a2611=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., 7eae297055c9fa234af9f1ed424b842e=testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e., 8f7c51a52b68bf2ed5bf01e5bd055060=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.} 2024-11-17T01:57:07,541 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:07,741 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:07,910 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815/.tmp/testFamily/d0147f7200764a97bad6764dbe9a87cc 2024-11-17T01:57:07,928 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815/.tmp/testFamily/d0147f7200764a97bad6764dbe9a87cc as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815/testFamily/d0147f7200764a97bad6764dbe9a87cc 2024-11-17T01:57:07,937 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815/testFamily/d0147f7200764a97bad6764dbe9a87cc, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:07,938 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4bf2d331570c103643c58e369af12815 in 450ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:07,942 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 4bf2d331570c103643c58e369af12815, 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:07,943 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScannerAsync/4bf2d331570c103643c58e369af12815/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:07,944 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:07,944 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4bf2d331570c103643c58e369af12815: Waiting for close lock at 1731808627487Running coprocessor pre-close hooks at 1731808627487Disabling compacts and flushes for region at 1731808627487Disabling writes for close at 1731808627488 (+1 ms)Obtaining lock to block concurrent updates at 1731808627488Preparing flush snapshotting stores in 4bf2d331570c103643c58e369af12815 at 1731808627488Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808627488Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. at 1731808627489 (+1 ms)Flushing 4bf2d331570c103643c58e369af12815/testFamily: creating writer at 1731808627489Flushing 4bf2d331570c103643c58e369af12815/testFamily: appending metadata at 1731808627502 (+13 ms)Flushing 4bf2d331570c103643c58e369af12815/testFamily: closing flushed file at 1731808627503 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542959c5: reopening flushed file at 1731808627926 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4bf2d331570c103643c58e369af12815 in 450ms, sequenceid=8, compaction requested=false at 1731808627938 (+12 ms)Writing region close event to WAL at 1731808627939 (+1 ms)Running coprocessor post-close hooks at 1731808627944 (+5 ms)Closed at 1731808627944 2024-11-17T01:57:07,944 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731808619365.4bf2d331570c103643c58e369af12815. 2024-11-17T01:57:07,945 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1f7df8e6f82328512994f55ec7a732b7, disabling compactions & flushes 2024-11-17T01:57:07,945 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:57:07,945 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:57:07,945 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. after waiting 0 ms 2024-11-17T01:57:07,945 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:57:07,945 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1f7df8e6f82328512994f55ec7a732b7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:07,965 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7/.tmp/testFamily/dfc6d27016b84d4cb90e34f67cbc7da0 is 61, key is row-0/testFamily:testQualifier/1731808561467/Put/seqid=0 2024-11-17T01:57:07,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741864_1040 (size=5049) 2024-11-17T01:57:08,142 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:08,342 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 1f7df8e6f82328512994f55ec7a732b7, 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:08,372 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7/.tmp/testFamily/dfc6d27016b84d4cb90e34f67cbc7da0 2024-11-17T01:57:08,387 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7/.tmp/testFamily/dfc6d27016b84d4cb90e34f67cbc7da0 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7/testFamily/dfc6d27016b84d4cb90e34f67cbc7da0 2024-11-17T01:57:08,394 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7/testFamily/dfc6d27016b84d4cb90e34f67cbc7da0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:08,395 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1f7df8e6f82328512994f55ec7a732b7 in 450ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:08,400 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1f7df8e6f82328512994f55ec7a732b7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:08,400 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:57:08,400 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1f7df8e6f82328512994f55ec7a732b7: Waiting for close lock at 1731808627945Running coprocessor pre-close hooks at 1731808627945Disabling compacts and flushes for region at 1731808627945Disabling writes for close at 1731808627945Obtaining lock to block concurrent updates at 1731808627945Preparing flush snapshotting stores in 1f7df8e6f82328512994f55ec7a732b7 at 1731808627945Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808627945Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. at 1731808627946 (+1 ms)Flushing 1f7df8e6f82328512994f55ec7a732b7/testFamily: creating writer at 1731808627946Flushing 1f7df8e6f82328512994f55ec7a732b7/testFamily: appending metadata at 1731808627965 (+19 ms)Flushing 1f7df8e6f82328512994f55ec7a732b7/testFamily: closing flushed file at 1731808627965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d664262: reopening flushed file at 1731808628385 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1f7df8e6f82328512994f55ec7a732b7 in 450ms, sequenceid=8, compaction requested=false at 1731808628395 (+10 ms)Writing region close event to WAL at 1731808628396 (+1 ms)Running coprocessor post-close hooks at 1731808628400 (+4 ms)Closed at 1731808628400 2024-11-17T01:57:08,401 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731808560393.1f7df8e6f82328512994f55ec7a732b7. 2024-11-17T01:57:08,401 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e90e10a3b126b34ec9c5c884997a2611, disabling compactions & flushes 2024-11-17T01:57:08,401 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:57:08,401 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:57:08,401 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. after waiting 0 ms 2024-11-17T01:57:08,401 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:57:08,401 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e90e10a3b126b34ec9c5c884997a2611 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:08,419 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611/.tmp/testFamily/586061a21e0e4f12a5267ca9ab1eb4a6 is 61, key is row-0/testFamily:testQualifier/1731808585898/Put/seqid=0 2024-11-17T01:57:08,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741865_1041 (size=5049) 2024-11-17T01:57:08,543 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-17T01:57:08,543 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1325): Online Regions={e90e10a3b126b34ec9c5c884997a2611=testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., 7eae297055c9fa234af9f1ed424b842e=testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e., 8f7c51a52b68bf2ed5bf01e5bd055060=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.} 2024-11-17T01:57:08,543 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:08,744 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060, e90e10a3b126b34ec9c5c884997a2611 2024-11-17T01:57:08,826 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611/.tmp/testFamily/586061a21e0e4f12a5267ca9ab1eb4a6 2024-11-17T01:57:08,841 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611/.tmp/testFamily/586061a21e0e4f12a5267ca9ab1eb4a6 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611/testFamily/586061a21e0e4f12a5267ca9ab1eb4a6 2024-11-17T01:57:08,848 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611/testFamily/586061a21e0e4f12a5267ca9ab1eb4a6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:08,849 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e90e10a3b126b34ec9c5c884997a2611 in 448ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:08,854 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnOpenScanner/e90e10a3b126b34ec9c5c884997a2611/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:08,855 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:57:08,855 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e90e10a3b126b34ec9c5c884997a2611: Waiting for close lock at 1731808628401Running coprocessor pre-close hooks at 1731808628401Disabling compacts and flushes for region at 1731808628401Disabling writes for close at 1731808628401Obtaining lock to block concurrent updates at 1731808628401Preparing flush snapshotting stores in e90e10a3b126b34ec9c5c884997a2611 at 1731808628401Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808628401Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. at 1731808628402 (+1 ms)Flushing e90e10a3b126b34ec9c5c884997a2611/testFamily: creating writer at 1731808628402Flushing e90e10a3b126b34ec9c5c884997a2611/testFamily: appending metadata at 1731808628419 (+17 ms)Flushing e90e10a3b126b34ec9c5c884997a2611/testFamily: closing flushed file at 1731808628419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15c10dbb: reopening flushed file at 1731808628840 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e90e10a3b126b34ec9c5c884997a2611 in 448ms, sequenceid=8, compaction requested=false at 1731808628849 (+9 ms)Writing region close event to WAL at 1731808628850 (+1 ms)Running coprocessor post-close hooks at 1731808628855 (+5 ms)Closed at 1731808628855 2024-11-17T01:57:08,855 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731808582862.e90e10a3b126b34ec9c5c884997a2611. 2024-11-17T01:57:08,855 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7eae297055c9fa234af9f1ed424b842e, disabling compactions & flushes 2024-11-17T01:57:08,855 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:57:08,855 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:57:08,855 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. after waiting 0 ms 2024-11-17T01:57:08,855 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:57:08,855 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7eae297055c9fa234af9f1ed424b842e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:08,872 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e/.tmp/testFamily/501b99c5a2034335a2be5f40ea4c7e98 is 61, key is row-0/testFamily:testQualifier/1731808592016/Put/seqid=0 2024-11-17T01:57:08,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741866_1042 (size=5049) 2024-11-17T01:57:08,944 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:57:09,144 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 7eae297055c9fa234af9f1ed424b842e, 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:57:09,279 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e/.tmp/testFamily/501b99c5a2034335a2be5f40ea4c7e98 2024-11-17T01:57:09,294 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e/.tmp/testFamily/501b99c5a2034335a2be5f40ea4c7e98 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e/testFamily/501b99c5a2034335a2be5f40ea4c7e98 2024-11-17T01:57:09,303 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e/testFamily/501b99c5a2034335a2be5f40ea4c7e98, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:09,304 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7eae297055c9fa234af9f1ed424b842e in 449ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:09,309 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/default/testNormalScanTimeoutOnNextAsync/7eae297055c9fa234af9f1ed424b842e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:09,310 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:57:09,310 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7eae297055c9fa234af9f1ed424b842e: Waiting for close lock at 1731808628855Running coprocessor pre-close hooks at 1731808628855Disabling compacts and flushes for region at 1731808628855Disabling writes for close at 1731808628855Obtaining lock to block concurrent updates at 1731808628855Preparing flush snapshotting stores in 7eae297055c9fa234af9f1ed424b842e at 1731808628855Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808628856 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. at 1731808628856Flushing 7eae297055c9fa234af9f1ed424b842e/testFamily: creating writer at 1731808628857 (+1 ms)Flushing 7eae297055c9fa234af9f1ed424b842e/testFamily: appending metadata at 1731808628872 (+15 ms)Flushing 7eae297055c9fa234af9f1ed424b842e/testFamily: closing flushed file at 1731808628872Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d5446bb: reopening flushed file at 1731808629292 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7eae297055c9fa234af9f1ed424b842e in 449ms, sequenceid=8, compaction requested=false at 1731808629304 (+12 ms)Writing region close event to WAL at 1731808629305 (+1 ms)Running coprocessor post-close hooks at 1731808629310 (+5 ms)Closed at 1731808629310 2024-11-17T01:57:09,310 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731808588939.7eae297055c9fa234af9f1ed424b842e. 2024-11-17T01:57:09,310 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8f7c51a52b68bf2ed5bf01e5bd055060, disabling compactions & flushes 2024-11-17T01:57:09,310 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:57:09,310 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:57:09,310 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. after waiting 0 ms 2024-11-17T01:57:09,310 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:57:09,311 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8f7c51a52b68bf2ed5bf01e5bd055060 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-17T01:57:09,327 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060/.tmp/testFamily/40a9b89044db4ea4a8ba7f2c39ce5fe5 is 61, key is row-0/testFamily:testQualifier/1731808564612/Put/seqid=0 2024-11-17T01:57:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741867_1043 (size=5049) 2024-11-17T01:57:09,344 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:57:09,545 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T01:57:09,545 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1325): Online Regions={8f7c51a52b68bf2ed5bf01e5bd055060=hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060.} 2024-11-17T01:57:09,545 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:57:09,734 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060/.tmp/testFamily/40a9b89044db4ea4a8ba7f2c39ce5fe5 2024-11-17T01:57:09,745 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1351): Waiting on 8f7c51a52b68bf2ed5bf01e5bd055060 2024-11-17T01:57:09,748 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060/.tmp/testFamily/40a9b89044db4ea4a8ba7f2c39ce5fe5 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060/testFamily/40a9b89044db4ea4a8ba7f2c39ce5fe5 2024-11-17T01:57:09,756 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060/testFamily/40a9b89044db4ea4a8ba7f2c39ce5fe5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-17T01:57:09,758 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8f7c51a52b68bf2ed5bf01e5bd055060 in 447ms, sequenceid=8, compaction requested=false 2024-11-17T01:57:09,763 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/data/hbase/testMetaScanTimeoutOnOpenScanner/8f7c51a52b68bf2ed5bf01e5bd055060/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-17T01:57:09,764 INFO [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:57:09,764 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8f7c51a52b68bf2ed5bf01e5bd055060: Waiting for close lock at 1731808629310Running coprocessor pre-close hooks at 1731808629310Disabling compacts and flushes for region at 1731808629310Disabling writes for close at 1731808629310Obtaining lock to block concurrent updates at 1731808629311 (+1 ms)Preparing flush snapshotting stores in 8f7c51a52b68bf2ed5bf01e5bd055060 at 1731808629311Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731808629311Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. at 1731808629311Flushing 8f7c51a52b68bf2ed5bf01e5bd055060/testFamily: creating writer at 1731808629312 (+1 ms)Flushing 8f7c51a52b68bf2ed5bf01e5bd055060/testFamily: appending metadata at 1731808629326 (+14 ms)Flushing 8f7c51a52b68bf2ed5bf01e5bd055060/testFamily: closing flushed file at 1731808629327 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b214ef: reopening flushed file at 1731808629746 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8f7c51a52b68bf2ed5bf01e5bd055060 in 447ms, sequenceid=8, compaction requested=false at 1731808629758 (+12 ms)Writing region close event to WAL at 1731808629758Running coprocessor post-close hooks at 1731808629764 (+6 ms)Closed at 1731808629764 2024-11-17T01:57:09,764 DEBUG [RS_CLOSE_REGION-regionserver/c6c0d7a0a7c0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731808561557.8f7c51a52b68bf2ed5bf01e5bd055060. 2024-11-17T01:57:09,946 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(976): stopping server c6c0d7a0a7c0,38643,1731808526242; all regions closed. 2024-11-17T01:57:09,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741834_1010 (size=19479) 2024-11-17T01:57:10,003 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/oldWALs 2024-11-17T01:57:10,003 INFO [RS:0;c6c0d7a0a7c0:38643 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C38643%2C1731808526242.meta:.meta(num 1731808559924) 2024-11-17T01:57:10,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741833_1009 (size=20031) 2024-11-17T01:57:10,008 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/WALs/c6c0d7a0a7c0,38643,1731808526242/c6c0d7a0a7c0%2C38643%2C1731808526242.1731808559559 not finished, retry = 0 2024-11-17T01:57:10,115 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/oldWALs 2024-11-17T01:57:10,115 INFO [RS:0;c6c0d7a0a7c0:38643 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6c0d7a0a7c0%2C38643%2C1731808526242:(num 1731808559559) 2024-11-17T01:57:10,116 DEBUG [RS:0;c6c0d7a0a7c0:38643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T01:57:10,116 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:57:10,116 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:57:10,117 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.ChoreService(370): Chore service for: regionserver/c6c0d7a0a7c0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T01:57:10,118 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:57:10,118 INFO [regionserver/c6c0d7a0a7c0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:57:10,121 INFO [RS:0;c6c0d7a0a7c0:38643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38643 2024-11-17T01:57:10,124 INFO [regionserver/c6c0d7a0a7c0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T01:57:10,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T01:57:10,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6c0d7a0a7c0,38643,1731808526242 2024-11-17T01:57:10,154 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:57:10,163 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6c0d7a0a7c0,38643,1731808526242] 2024-11-17T01:57:10,171 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6c0d7a0a7c0,38643,1731808526242 already deleted, retry=false 2024-11-17T01:57:10,171 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6c0d7a0a7c0,38643,1731808526242 expired; onlineServers=0 2024-11-17T01:57:10,171 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c6c0d7a0a7c0,33177,1731808525509' ***** 2024-11-17T01:57:10,172 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T01:57:10,172 INFO [M:0;c6c0d7a0a7c0:33177 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T01:57:10,172 INFO [M:0;c6c0d7a0a7c0:33177 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T01:57:10,172 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T01:57:10,173 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T01:57:10,173 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T01:57:10,173 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808529337 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.large.0-1731808529337,5,FailOnTimeoutGroup] 2024-11-17T01:57:10,173 DEBUG [master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808529339 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6c0d7a0a7c0:0:becomeActiveMaster-HFileCleaner.small.0-1731808529339,5,FailOnTimeoutGroup] 2024-11-17T01:57:10,173 INFO [M:0;c6c0d7a0a7c0:33177 {}] hbase.ChoreService(370): Chore service for: master/c6c0d7a0a7c0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T01:57:10,174 INFO [M:0;c6c0d7a0a7c0:33177 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T01:57:10,174 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] master.HMaster(1795): Stopping service threads 2024-11-17T01:57:10,174 INFO [M:0;c6c0d7a0a7c0:33177 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T01:57:10,175 INFO [M:0;c6c0d7a0a7c0:33177 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T01:57:10,176 INFO [M:0;c6c0d7a0a7c0:33177 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T01:57:10,176 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T01:57:10,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T01:57:10,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T01:57:10,179 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] zookeeper.ZKUtil(347): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T01:57:10,180 WARN [M:0;c6c0d7a0a7c0:33177 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T01:57:10,181 INFO [M:0;c6c0d7a0a7c0:33177 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/.lastflushedseqids 2024-11-17T01:57:10,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741868_1044 (size=649) 2024-11-17T01:57:10,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:57:10,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38643-0x10147ca8c3f0001, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:57:10,264 INFO [RS:0;c6c0d7a0a7c0:38643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:57:10,264 INFO [RS:0;c6c0d7a0a7c0:38643 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6c0d7a0a7c0,38643,1731808526242; zookeeper connection closed. 2024-11-17T01:57:10,265 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@41027393 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@41027393 2024-11-17T01:57:10,266 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T01:57:10,598 INFO [M:0;c6c0d7a0a7c0:33177 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T01:57:10,599 INFO [M:0;c6c0d7a0a7c0:33177 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T01:57:10,600 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T01:57:10,600 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:57:10,600 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:57:10,600 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T01:57:10,601 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:57:10,601 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.97 KB heapSize=190.84 KB 2024-11-17T01:57:10,623 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc390cc51be44dc2b809a0f433d7fdb4 is 82, key is hbase:meta,,1/info:regioninfo/1731808560013/Put/seqid=0 2024-11-17T01:57:10,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741869_1045 (size=5672) 2024-11-17T01:57:11,031 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc390cc51be44dc2b809a0f433d7fdb4 2024-11-17T01:57:11,062 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b6ec5ab5a35f46a7b1900abf0ac103b9 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731808561253/Put/seqid=0 2024-11-17T01:57:11,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741870_1046 (size=14067) 2024-11-17T01:57:11,172 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T01:57:11,470 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.41 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b6ec5ab5a35f46a7b1900abf0ac103b9 2024-11-17T01:57:11,482 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b6ec5ab5a35f46a7b1900abf0ac103b9 2024-11-17T01:57:11,500 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cda99a4cd6fe4a7aaf70ff1465a7a86e is 69, key is c6c0d7a0a7c0,38643,1731808526242/rs:state/1731808529387/Put/seqid=0 2024-11-17T01:57:11,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741871_1047 (size=5156) 2024-11-17T01:57:11,908 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cda99a4cd6fe4a7aaf70ff1465a7a86e 2024-11-17T01:57:11,921 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc390cc51be44dc2b809a0f433d7fdb4 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc390cc51be44dc2b809a0f433d7fdb4 2024-11-17T01:57:11,928 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc390cc51be44dc2b809a0f433d7fdb4, entries=8, sequenceid=317, filesize=5.5 K 2024-11-17T01:57:11,929 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b6ec5ab5a35f46a7b1900abf0ac103b9 as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b6ec5ab5a35f46a7b1900abf0ac103b9 2024-11-17T01:57:11,936 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b6ec5ab5a35f46a7b1900abf0ac103b9 2024-11-17T01:57:11,936 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b6ec5ab5a35f46a7b1900abf0ac103b9, entries=33, sequenceid=317, filesize=13.7 K 2024-11-17T01:57:11,938 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cda99a4cd6fe4a7aaf70ff1465a7a86e as hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cda99a4cd6fe4a7aaf70ff1465a7a86e 2024-11-17T01:57:11,945 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cda99a4cd6fe4a7aaf70ff1465a7a86e, entries=1, sequenceid=317, filesize=5.0 K 2024-11-17T01:57:11,946 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.55 KB/195120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1345ms, sequenceid=317, compaction requested=false 2024-11-17T01:57:11,948 INFO [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T01:57:11,948 DEBUG [M:0;c6c0d7a0a7c0:33177 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731808630600Disabling compacts and flushes for region at 1731808630600Disabling writes for close at 1731808630600Obtaining lock to block concurrent updates at 1731808630602 (+2 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731808630602Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166879, getHeapSize=195360, getOffHeapSize=0, getCellsCount=364 at 1731808630603 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731808630603Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731808630604 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731808630623 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731808630623Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731808631044 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731808631062 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731808631062Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731808631483 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731808631499 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731808631500 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ea80c21: reopening flushed file at 1731808631920 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e68efa: reopening flushed file at 1731808631928 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ae29c2: reopening flushed file at 1731808631936 (+8 ms)Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.55 KB/195120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1345ms, sequenceid=317, compaction requested=false at 1731808631946 (+10 ms)Writing region close event to WAL at 1731808631948 (+2 ms)Closed at 1731808631948 2024-11-17T01:57:11,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45353 is added to blk_1073741830_1006 (size=188091) 2024-11-17T01:57:11,951 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/611cd2b9-be4d-93ae-4a38-f6de72fd7067/MasterData/WALs/c6c0d7a0a7c0,33177,1731808525509/c6c0d7a0a7c0%2C33177%2C1731808525509.1731808528372 not finished, retry = 0 2024-11-17T01:57:12,052 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T01:57:12,053 INFO [M:0;c6c0d7a0a7c0:33177 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T01:57:12,053 INFO [M:0;c6c0d7a0a7c0:33177 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33177 2024-11-17T01:57:12,053 INFO [M:0;c6c0d7a0a7c0:33177 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T01:57:12,213 INFO [M:0;c6c0d7a0a7c0:33177 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T01:57:12,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:57:12,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33177-0x10147ca8c3f0000, quorum=127.0.0.1:60070, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T01:57:12,221 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bc78d53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T01:57:12,224 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T01:57:12,224 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T01:57:12,225 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T01:57:12,225 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/hadoop.log.dir/,STOPPED} 2024-11-17T01:57:12,227 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T01:57:12,227 WARN [BP-1067243206-172.17.0.2-1731808521450 heartbeating to localhost/127.0.0.1:40955 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T01:57:12,227 WARN [BP-1067243206-172.17.0.2-1731808521450 heartbeating to localhost/127.0.0.1:40955 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1067243206-172.17.0.2-1731808521450 (Datanode Uuid d98683ea-0e10-498e-b4c6-87db7eceabd9) service to localhost/127.0.0.1:40955 2024-11-17T01:57:12,227 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T01:57:12,229 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d/data/data1/current/BP-1067243206-172.17.0.2-1731808521450 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T01:57:12,229 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/cluster_09469881-7c66-c00d-5432-0aef14eb038d/data/data2/current/BP-1067243206-172.17.0.2-1731808521450 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T01:57:12,229 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T01:57:12,238 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@562f0d0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T01:57:12,238 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T01:57:12,238 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T01:57:12,239 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T01:57:12,239 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/62ad5ea8-4ea3-a98a-2cbd-eb67de5a0fb3/hadoop.log.dir/,STOPPED} 2024-11-17T01:57:12,246 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T01:57:12,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down