2024-12-06 15:01:58,257 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-06 15:01:58,271 main DEBUG Took 0.011242 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 15:01:58,271 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 15:01:58,272 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 15:01:58,277 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 15:01:58,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,284 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 15:01:58,294 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,296 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,297 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,298 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,298 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,299 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,300 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,301 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,301 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,302 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,303 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,303 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 15:01:58,304 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,304 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 15:01:58,305 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 15:01:58,306 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 15:01:58,308 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 15:01:58,309 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 15:01:58,310 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 15:01:58,310 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 15:01:58,319 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 15:01:58,321 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 15:01:58,323 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 15:01:58,323 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 15:01:58,323 main DEBUG createAppenders(={Console}) 2024-12-06 15:01:58,324 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-06 15:01:58,324 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-06 15:01:58,325 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-06 15:01:58,325 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 15:01:58,326 main DEBUG OutputStream closed 2024-12-06 15:01:58,326 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 15:01:58,326 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 15:01:58,326 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-06 15:01:58,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 15:01:58,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 15:01:58,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 15:01:58,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 15:01:58,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 15:01:58,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 15:01:58,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 15:01:58,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 15:01:58,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 15:01:58,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 15:01:58,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 15:01:58,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 15:01:58,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 15:01:58,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 15:01:58,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 15:01:58,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 15:01:58,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 15:01:58,398 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 15:01:58,400 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 15:01:58,400 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-06 15:01:58,400 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 15:01:58,401 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-06T15:01:58,414 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-06 15:01:58,417 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 15:01:58,418 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T15:01:58,639 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b 2024-12-06T15:01:58,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-06T15:01:58,668 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207, deleteOnExit=true 2024-12-06T15:01:58,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-06T15:01:58,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/test.cache.data in system properties and HBase conf 2024-12-06T15:01:58,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T15:01:58,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/hadoop.log.dir in system properties and HBase conf 2024-12-06T15:01:58,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T15:01:58,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T15:01:58,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T15:01:58,755 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T15:01:58,839 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T15:01:58,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T15:01:58,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T15:01:58,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T15:01:58,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T15:01:58,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T15:01:58,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T15:01:58,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T15:01:58,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T15:01:58,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T15:01:58,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/nfs.dump.dir in system properties and HBase conf 2024-12-06T15:01:58,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/java.io.tmpdir in system properties and HBase conf 2024-12-06T15:01:58,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T15:01:58,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T15:01:58,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T15:01:59,984 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-06T15:02:00,068 INFO [Time-limited test {}] log.Log(170): Logging initialized @2444ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-06T15:02:00,141 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:02:00,216 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:02:00,238 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:02:00,238 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:02:00,240 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T15:02:00,256 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:02:00,260 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:02:00,261 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:02:00,459 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/java.io.tmpdir/jetty-localhost-43973-hadoop-hdfs-3_4_1-tests_jar-_-any-14778472981467572281/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T15:02:00,466 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:43973} 2024-12-06T15:02:00,467 INFO [Time-limited test {}] server.Server(415): Started @2843ms 2024-12-06T15:02:01,233 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T15:02:01,240 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T15:02:01,243 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T15:02:01,243 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T15:02:01,243 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T15:02:01,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/hadoop.log.dir/,AVAILABLE} 2024-12-06T15:02:01,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T15:02:01,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/java.io.tmpdir/jetty-localhost-38033-hadoop-hdfs-3_4_1-tests_jar-_-any-16740254009918920991/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:02:01,354 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:38033} 2024-12-06T15:02:01,355 INFO [Time-limited test {}] server.Server(415): Started @3731ms 2024-12-06T15:02:01,403 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T15:02:02,511 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207/data/data1/current/BP-1257781960-172.17.0.2-1733497319399/current, will proceed with Du for space computation calculation, 2024-12-06T15:02:02,511 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207/data/data2/current/BP-1257781960-172.17.0.2-1733497319399/current, will proceed with Du for space computation calculation, 2024-12-06T15:02:02,555 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T15:02:02,609 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88b464348a70d460 with lease ID 0x27277c6d098adf05: Processing first storage report for DS-5922685c-1ea0-43b0-9c65-d06f445d08e8 from datanode DatanodeRegistration(127.0.0.1:42777, datanodeUuid=fd935b72-4700-4b0b-880c-d48f79917280, infoPort=35781, infoSecurePort=0, ipcPort=40247, storageInfo=lv=-57;cid=testClusterID;nsid=340831708;c=1733497319399) 2024-12-06T15:02:02,610 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88b464348a70d460 with lease ID 0x27277c6d098adf05: from storage DS-5922685c-1ea0-43b0-9c65-d06f445d08e8 node DatanodeRegistration(127.0.0.1:42777, datanodeUuid=fd935b72-4700-4b0b-880c-d48f79917280, infoPort=35781, infoSecurePort=0, ipcPort=40247, storageInfo=lv=-57;cid=testClusterID;nsid=340831708;c=1733497319399), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-06T15:02:02,611 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88b464348a70d460 with lease ID 0x27277c6d098adf05: Processing first storage report for DS-aca98e08-bc63-4007-b088-c695fbc1a19d from datanode DatanodeRegistration(127.0.0.1:42777, datanodeUuid=fd935b72-4700-4b0b-880c-d48f79917280, infoPort=35781, infoSecurePort=0, ipcPort=40247, storageInfo=lv=-57;cid=testClusterID;nsid=340831708;c=1733497319399) 2024-12-06T15:02:02,611 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88b464348a70d460 with lease ID 0x27277c6d098adf05: from storage DS-aca98e08-bc63-4007-b088-c695fbc1a19d node DatanodeRegistration(127.0.0.1:42777, datanodeUuid=fd935b72-4700-4b0b-880c-d48f79917280, infoPort=35781, infoSecurePort=0, ipcPort=40247, storageInfo=lv=-57;cid=testClusterID;nsid=340831708;c=1733497319399), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T15:02:02,694 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b 2024-12-06T15:02:02,796 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207/zookeeper_0, clientPort=50293, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T15:02:02,810 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50293 2024-12-06T15:02:02,824 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:02:02,828 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:02:03,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741825_1001 (size=7) 2024-12-06T15:02:03,508 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf with version=8 2024-12-06T15:02:03,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/hbase-staging 2024-12-06T15:02:03,579 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T15:02:03,801 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bb7ea3e46d9c:0 server-side Connection retries=9 2024-12-06T15:02:03,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T15:02:03,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T15:02:03,814 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T15:02:03,815 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T15:02:03,815 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T15:02:03,961 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T15:02:04,017 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T15:02:04,025 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T15:02:04,029 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T15:02:04,052 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 123602 (auto-detected) 2024-12-06T15:02:04,054 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T15:02:04,078 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43691 2024-12-06T15:02:04,106 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43691 connecting to ZooKeeper ensemble=127.0.0.1:50293 2024-12-06T15:02:04,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436910x0, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T15:02:04,216 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43691-0x101ab29f8bc0000 connected 2024-12-06T15:02:04,306 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:02:04,309 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:02:04,320 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T15:02:04,325 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf, hbase.cluster.distributed=false 2024-12-06T15:02:04,351 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T15:02:04,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43691 2024-12-06T15:02:04,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43691 2024-12-06T15:02:04,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43691 2024-12-06T15:02:04,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43691 2024-12-06T15:02:04,367 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43691 2024-12-06T15:02:04,467 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bb7ea3e46d9c:0 server-side Connection retries=9 2024-12-06T15:02:04,468 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T15:02:04,468 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T15:02:04,469 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T15:02:04,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T15:02:04,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T15:02:04,471 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T15:02:04,473 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T15:02:04,474 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45829 2024-12-06T15:02:04,475 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45829 connecting to ZooKeeper ensemble=127.0.0.1:50293 2024-12-06T15:02:04,476 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:02:04,480 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:02:04,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458290x0, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T15:02:04,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:458290x0, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T15:02:04,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45829-0x101ab29f8bc0001 connected 2024-12-06T15:02:04,509 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T15:02:04,516 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T15:02:04,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T15:02:04,523 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T15:02:04,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45829 2024-12-06T15:02:04,526 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45829 2024-12-06T15:02:04,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45829 2024-12-06T15:02:04,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45829 2024-12-06T15:02:04,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45829 2024-12-06T15:02:04,546 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bb7ea3e46d9c:43691 2024-12-06T15:02:04,547 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:04,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T15:02:04,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T15:02:04,580 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:04,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:04,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T15:02:04,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:04,642 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T15:02:04,644 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bb7ea3e46d9c,43691,1733497323635 from backup master directory 2024-12-06T15:02:04,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:04,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T15:02:04,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T15:02:04,662 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T15:02:04,663 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:04,664 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T15:02:04,666 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T15:02:04,714 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/hbase.id] with ID: 1523a5ba-684a-40c5-b7b5-68c3c531e923 2024-12-06T15:02:04,714 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/.tmp/hbase.id 2024-12-06T15:02:04,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741826_1002 (size=42) 2024-12-06T15:02:05,127 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/.tmp/hbase.id]:[hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/hbase.id] 2024-12-06T15:02:05,175 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T15:02:05,180 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T15:02:05,197 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-06T15:02:05,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:05,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:05,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741827_1003 (size=196) 2024-12-06T15:02:05,689 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:02:05,691 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T15:02:05,703 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:02:05,707 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:02:05,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741828_1004 (size=1189) 2024-12-06T15:02:05,749 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store 2024-12-06T15:02:05,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741829_1005 (size=34) 2024-12-06T15:02:05,769 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T15:02:05,772 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:05,774 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T15:02:05,774 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:02:05,774 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:02:05,776 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T15:02:05,776 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:02:05,776 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:02:05,778 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733497325773Disabling compacts and flushes for region at 1733497325774 (+1 ms)Disabling writes for close at 1733497325776 (+2 ms)Writing region close event to WAL at 1733497325776Closed at 1733497325776 2024-12-06T15:02:05,780 WARN [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/.initializing 2024-12-06T15:02:05,780 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/WALs/bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:05,789 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:02:05,804 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C43691%2C1733497323635, suffix=, logDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/WALs/bb7ea3e46d9c,43691,1733497323635, archiveDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/oldWALs, maxLogs=10 2024-12-06T15:02:05,825 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/WALs/bb7ea3e46d9c,43691,1733497323635/bb7ea3e46d9c%2C43691%2C1733497323635.1733497325809, exclude list is [], retry=0 2024-12-06T15:02:05,839 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42777,DS-5922685c-1ea0-43b0-9c65-d06f445d08e8,DISK] 2024-12-06T15:02:05,841 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-06T15:02:05,890 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/WALs/bb7ea3e46d9c,43691,1733497323635/bb7ea3e46d9c%2C43691%2C1733497323635.1733497325809 2024-12-06T15:02:05,891 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35781:35781)] 2024-12-06T15:02:05,892 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:02:05,892 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:05,895 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,896 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T15:02:05,954 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:05,956 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:05,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T15:02:05,960 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:05,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:02:05,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T15:02:05,965 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:05,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:02:05,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T15:02:05,971 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:05,972 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:02:05,972 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,976 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,977 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,982 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,983 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,986 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T15:02:05,990 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T15:02:05,996 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:02:05,997 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63922935, jitterRate=-0.04747404158115387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T15:02:06,005 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733497325906Initializing all the Stores at 1733497325909 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733497325909Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497325910 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497325910Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497325910Cleaning up temporary data from old regions at 1733497325983 (+73 ms)Region opened successfully at 1733497326004 (+21 ms) 2024-12-06T15:02:06,006 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T15:02:06,039 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77247943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T15:02:06,068 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T15:02:06,079 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T15:02:06,080 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T15:02:06,082 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T15:02:06,084 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T15:02:06,089 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-06T15:02:06,089 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T15:02:06,113 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T15:02:06,123 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T15:02:06,135 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T15:02:06,138 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T15:02:06,139 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T15:02:06,151 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T15:02:06,155 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T15:02:06,160 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T15:02:06,173 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T15:02:06,175 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T15:02:06,187 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T15:02:06,205 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T15:02:06,215 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T15:02:06,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T15:02:06,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T15:02:06,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:06,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:06,234 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bb7ea3e46d9c,43691,1733497323635, sessionid=0x101ab29f8bc0000, setting cluster-up flag (Was=false) 2024-12-06T15:02:06,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:06,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:06,293 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T15:02:06,295 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:06,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:06,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:06,367 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T15:02:06,369 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:06,377 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T15:02:06,444 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(746): ClusterId : 1523a5ba-684a-40c5-b7b5-68c3c531e923 2024-12-06T15:02:06,448 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T15:02:06,456 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T15:02:06,463 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T15:02:06,463 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T15:02:06,469 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T15:02:06,478 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T15:02:06,480 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T15:02:06,481 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49139dd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bb7ea3e46d9c/172.17.0.2:0 2024-12-06T15:02:06,487 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bb7ea3e46d9c,43691,1733497323635 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T15:02:06,497 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T15:02:06,500 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T15:02:06,500 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T15:02:06,501 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=5, maxPoolSize=5 2024-12-06T15:02:06,501 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bb7ea3e46d9c:0, corePoolSize=10, maxPoolSize=10 2024-12-06T15:02:06,501 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,501 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bb7ea3e46d9c:45829 2024-12-06T15:02:06,501 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T15:02:06,501 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,505 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T15:02:06,505 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T15:02:06,505 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T15:02:06,508 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(2659): reportForDuty to master=bb7ea3e46d9c,43691,1733497323635 with port=45829, startcode=1733497324439 2024-12-06T15:02:06,509 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T15:02:06,510 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T15:02:06,512 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733497356512 2024-12-06T15:02:06,514 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T15:02:06,515 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T15:02:06,517 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:06,517 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T15:02:06,519 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T15:02:06,520 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T15:02:06,520 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T15:02:06,521 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T15:02:06,521 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T15:02:06,522 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,528 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T15:02:06,530 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T15:02:06,531 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T15:02:06,533 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T15:02:06,533 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T15:02:06,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741831_1007 (size=1321) 2024-12-06T15:02:06,535 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733497326534,5,FailOnTimeoutGroup] 2024-12-06T15:02:06,538 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733497326536,5,FailOnTimeoutGroup] 2024-12-06T15:02:06,538 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,538 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T15:02:06,540 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,540 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,541 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T15:02:06,542 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:02:06,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33785, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T15:02:06,571 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43691 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741832_1008 (size=32) 2024-12-06T15:02:06,576 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43691 {}] master.ServerManager(517): Registering regionserver=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:06,592 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:02:06,593 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35547 2024-12-06T15:02:06,593 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T15:02:06,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T15:02:06,605 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] zookeeper.ZKUtil(111): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:06,605 WARN [RS:0;bb7ea3e46d9c:45829 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T15:02:06,605 INFO [RS:0;bb7ea3e46d9c:45829 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:02:06,606 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/WALs/bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:06,608 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bb7ea3e46d9c,45829,1733497324439] 2024-12-06T15:02:06,628 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T15:02:06,640 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T15:02:06,645 INFO [RS:0;bb7ea3e46d9c:45829 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T15:02:06,645 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,646 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T15:02:06,651 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T15:02:06,652 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,652 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,653 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,653 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,653 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,653 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,653 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=2, maxPoolSize=2 2024-12-06T15:02:06,653 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,653 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,654 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,654 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,654 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,654 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bb7ea3e46d9c:0, corePoolSize=1, maxPoolSize=1 2024-12-06T15:02:06,654 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T15:02:06,654 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bb7ea3e46d9c:0, corePoolSize=3, maxPoolSize=3 2024-12-06T15:02:06,655 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,655 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,655 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,655 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,656 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,656 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,45829,1733497324439-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T15:02:06,671 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T15:02:06,673 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,45829,1733497324439-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,673 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,673 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.Replication(171): bb7ea3e46d9c,45829,1733497324439 started 2024-12-06T15:02:06,688 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:06,689 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1482): Serving as bb7ea3e46d9c,45829,1733497324439, RpcServer on bb7ea3e46d9c/172.17.0.2:45829, sessionid=0x101ab29f8bc0001 2024-12-06T15:02:06,689 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T15:02:06,690 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:06,690 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,45829,1733497324439' 2024-12-06T15:02:06,690 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T15:02:06,691 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T15:02:06,692 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T15:02:06,692 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T15:02:06,692 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:06,692 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bb7ea3e46d9c,45829,1733497324439' 2024-12-06T15:02:06,692 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T15:02:06,693 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T15:02:06,694 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T15:02:06,694 INFO [RS:0;bb7ea3e46d9c:45829 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T15:02:06,694 INFO [RS:0;bb7ea3e46d9c:45829 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T15:02:06,978 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:06,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T15:02:06,984 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T15:02:06,984 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:06,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:06,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T15:02:06,989 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T15:02:06,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:06,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:06,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T15:02:06,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T15:02:06,995 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:06,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:06,997 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T15:02:07,000 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T15:02:07,000 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:07,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:07,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T15:02:07,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740 2024-12-06T15:02:07,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740 2024-12-06T15:02:07,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T15:02:07,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T15:02:07,008 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T15:02:07,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T15:02:07,015 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:02:07,016 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66682262, jitterRate=-0.006356865167617798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T15:02:07,017 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733497326978Initializing all the Stores at 1733497326980 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733497326980Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733497326981 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497326981Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733497326981Cleaning up temporary data from old regions at 1733497327007 (+26 ms)Region opened successfully at 1733497327017 (+10 ms) 2024-12-06T15:02:07,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T15:02:07,018 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T15:02:07,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T15:02:07,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T15:02:07,018 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T15:02:07,019 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T15:02:07,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733497327018Disabling compacts and flushes for region at 1733497327018Disabling writes for close at 1733497327018Writing region close event to WAL at 1733497327019 (+1 ms)Closed at 1733497327019 2024-12-06T15:02:07,023 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T15:02:07,023 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T15:02:07,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T15:02:07,037 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T15:02:07,040 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T15:02:07,191 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:08,192 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:09,193 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:10,194 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:11,195 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:12,196 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:13,197 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:14,014 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T15:02:14,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-06T15:02:14,198 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:15,199 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:16,200 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:17,201 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:18,202 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:19,203 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:20,204 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:21,205 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:22,206 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:23,207 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:24,207 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:25,208 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:26,210 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:27,211 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:28,212 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:29,213 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:30,214 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:31,215 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:32,216 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:32,690 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T15:02:33,217 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:34,218 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:35,219 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:36,220 WARN [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T15:02:36,708 INFO [RS:0;bb7ea3e46d9c:45829 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T15:02:36,712 INFO [RS:0;bb7ea3e46d9c:45829 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C45829%2C1733497324439, suffix=, logDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/WALs/bb7ea3e46d9c,45829,1733497324439, archiveDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/oldWALs, maxLogs=32 2024-12-06T15:02:36,725 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/WALs/bb7ea3e46d9c,45829,1733497324439/bb7ea3e46d9c%2C45829%2C1733497324439.1733497356714, exclude list is [], retry=0 2024-12-06T15:02:36,728 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42777,DS-5922685c-1ea0-43b0-9c65-d06f445d08e8,DISK] 2024-12-06T15:02:36,771 INFO [RS:0;bb7ea3e46d9c:45829 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/WALs/bb7ea3e46d9c,45829,1733497324439/bb7ea3e46d9c%2C45829%2C1733497324439.1733497356714 2024-12-06T15:02:36,772 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35781:35781)] 2024-12-06T15:02:36,974 DEBUG [bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-06T15:02:36,987 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:36,992 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,45829,1733497324439, state=OPENING 2024-12-06T15:02:37,012 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T15:02:37,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:37,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:02:37,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T15:02:37,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T15:02:37,027 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T15:02:37,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:02:37,210 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T15:02:37,214 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32921, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T15:02:37,225 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T15:02:37,225 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T15:02:37,226 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T15:02:37,229 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bb7ea3e46d9c%2C45829%2C1733497324439.meta, suffix=.meta, logDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/WALs/bb7ea3e46d9c,45829,1733497324439, archiveDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/oldWALs, maxLogs=32 2024-12-06T15:02:37,242 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/WALs/bb7ea3e46d9c,45829,1733497324439/bb7ea3e46d9c%2C45829%2C1733497324439.meta.1733497357230.meta, exclude list is [], retry=0 2024-12-06T15:02:37,246 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42777,DS-5922685c-1ea0-43b0-9c65-d06f445d08e8,DISK] 2024-12-06T15:02:37,249 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/WALs/bb7ea3e46d9c,45829,1733497324439/bb7ea3e46d9c%2C45829%2C1733497324439.meta.1733497357230.meta 2024-12-06T15:02:37,249 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35781:35781)] 2024-12-06T15:02:37,250 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:02:37,251 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T15:02:37,253 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T15:02:37,257 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T15:02:37,261 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T15:02:37,262 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:37,262 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T15:02:37,262 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T15:02:37,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T15:02:37,266 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T15:02:37,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:37,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:37,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T15:02:37,269 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T15:02:37,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:37,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:37,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T15:02:37,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T15:02:37,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:37,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:37,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T15:02:37,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T15:02:37,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:37,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T15:02:37,276 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T15:02:37,278 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740 2024-12-06T15:02:37,281 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740 2024-12-06T15:02:37,284 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T15:02:37,284 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T15:02:37,285 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T15:02:37,287 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T15:02:37,289 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62662396, jitterRate=-0.0662575364112854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T15:02:37,289 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T15:02:37,291 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733497357262Writing region info on filesystem at 1733497357263 (+1 ms)Initializing all the Stores at 1733497357264 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733497357264Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733497357265 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497357265Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733497357265Cleaning up temporary data from old regions at 1733497357284 (+19 ms)Running coprocessor post-open hooks at 1733497357289 (+5 ms)Region opened successfully at 1733497357290 (+1 ms) 2024-12-06T15:02:37,297 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733497357200 2024-12-06T15:02:37,308 DEBUG [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T15:02:37,308 INFO [RS_OPEN_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T15:02:37,311 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:37,313 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bb7ea3e46d9c,45829,1733497324439, state=OPEN 2024-12-06T15:02:37,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T15:02:37,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T15:02:37,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T15:02:37,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T15:02:37,392 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:37,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T15:02:37,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bb7ea3e46d9c,45829,1733497324439 in 362 msec 2024-12-06T15:02:37,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T15:02:37,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.3740 sec 2024-12-06T15:02:37,410 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T15:02:37,410 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T15:02:37,429 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T15:02:37,431 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=-1] 2024-12-06T15:02:37,451 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T15:02:37,453 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58687, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T15:02:37,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0640 sec 2024-12-06T15:02:37,471 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733497357471, completionTime=-1 2024-12-06T15:02:37,474 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-06T15:02:37,474 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T15:02:37,499 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-06T15:02:37,499 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733497417499 2024-12-06T15:02:37,499 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733497477499 2024-12-06T15:02:37,499 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-06T15:02:37,501 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,43691,1733497323635-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:37,501 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,43691,1733497323635-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:37,501 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,43691,1733497323635-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:37,503 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bb7ea3e46d9c:43691, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:37,503 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:37,504 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:37,533 DEBUG [master/bb7ea3e46d9c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T15:02:37,552 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 32.889sec 2024-12-06T15:02:37,553 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T15:02:37,555 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T15:02:37,555 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T15:02:37,556 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T15:02:37,556 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T15:02:37,557 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,43691,1733497323635-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T15:02:37,557 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,43691,1733497323635-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T15:02:37,564 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T15:02:37,565 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T15:02:37,565 INFO [master/bb7ea3e46d9c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bb7ea3e46d9c,43691,1733497323635-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T15:02:37,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c8d99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T15:02:37,585 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T15:02:37,585 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T15:02:37,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,43691,-1 for getting cluster id 2024-12-06T15:02:37,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T15:02:37,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1523a5ba-684a-40c5-b7b5-68c3c531e923' 2024-12-06T15:02:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T15:02:37,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1523a5ba-684a-40c5-b7b5-68c3c531e923" 2024-12-06T15:02:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2a5478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T15:02:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,43691,-1] 2024-12-06T15:02:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T15:02:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:02:37,608 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T15:02:37,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8bff9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T15:02:37,611 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T15:02:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=-1] 2024-12-06T15:02:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T15:02:37,621 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T15:02:37,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:37,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228e3a11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T15:02:37,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bb7ea3e46d9c,43691,-1 for getting cluster id 2024-12-06T15:02:37,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T15:02:37,648 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1523a5ba-684a-40c5-b7b5-68c3c531e923' 2024-12-06T15:02:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T15:02:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1523a5ba-684a-40c5-b7b5-68c3c531e923" 2024-12-06T15:02:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459fdb13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T15:02:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bb7ea3e46d9c,43691,-1] 2024-12-06T15:02:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T15:02:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:02:37,652 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T15:02:37,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e126e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T15:02:37,668 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=6693 2024-12-06T15:02:37,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T15:02:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:02:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10fe7214 2024-12-06T15:02:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T15:02:37,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T15:02:37,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:02:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-06T15:02:37,703 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:02:37,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-06T15:02:37,705 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:37,707 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:02:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T15:02:37,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741835_1011 (size=428) 2024-12-06T15:02:38,127 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b49cda5460cc4f0dddbd13c60efce4f7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:02:38,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741836_1012 (size=79) 2024-12-06T15:02:38,138 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:38,138 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing b49cda5460cc4f0dddbd13c60efce4f7, disabling compactions & flushes 2024-12-06T15:02:38,139 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:02:38,139 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:02:38,139 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. after waiting 0 ms 2024-12-06T15:02:38,139 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:02:38,139 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:02:38,139 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for b49cda5460cc4f0dddbd13c60efce4f7: Waiting for close lock at 1733497358138Disabling compacts and flushes for region at 1733497358138Disabling writes for close at 1733497358139 (+1 ms)Writing region close event to WAL at 1733497358139Closed at 1733497358139 2024-12-06T15:02:38,142 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:02:38,148 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733497358142"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497358142"}]},"ts":"1733497358142"} 2024-12-06T15:02:38,152 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:02:38,154 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:02:38,157 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497358154"}]},"ts":"1733497358154"} 2024-12-06T15:02:38,161 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-06T15:02:38,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=b49cda5460cc4f0dddbd13c60efce4f7, ASSIGN}] 2024-12-06T15:02:38,165 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=b49cda5460cc4f0dddbd13c60efce4f7, ASSIGN 2024-12-06T15:02:38,167 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=b49cda5460cc4f0dddbd13c60efce4f7, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:02:38,319 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b49cda5460cc4f0dddbd13c60efce4f7, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:38,327 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=b49cda5460cc4f0dddbd13c60efce4f7, ASSIGN because future has completed 2024-12-06T15:02:38,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b49cda5460cc4f0dddbd13c60efce4f7, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:02:38,491 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:02:38,491 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b49cda5460cc4f0dddbd13c60efce4f7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:02:38,492 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,492 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:38,492 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,492 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,496 INFO [StoreOpener-b49cda5460cc4f0dddbd13c60efce4f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,499 INFO [StoreOpener-b49cda5460cc4f0dddbd13c60efce4f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b49cda5460cc4f0dddbd13c60efce4f7 columnFamilyName testFamily 2024-12-06T15:02:38,499 DEBUG [StoreOpener-b49cda5460cc4f0dddbd13c60efce4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:38,500 INFO [StoreOpener-b49cda5460cc4f0dddbd13c60efce4f7-1 {}] regionserver.HStore(327): Store=b49cda5460cc4f0dddbd13c60efce4f7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:02:38,500 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,502 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,503 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,503 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,503 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,506 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,509 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:02:38,510 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b49cda5460cc4f0dddbd13c60efce4f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59088712, jitterRate=-0.11950957775115967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:02:38,510 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:02:38,511 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b49cda5460cc4f0dddbd13c60efce4f7: Running coprocessor pre-open hook at 1733497358492Writing region info on filesystem at 1733497358492Initializing all the Stores at 1733497358495 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497358495Cleaning up temporary data from old regions at 1733497358503 (+8 ms)Running coprocessor post-open hooks at 1733497358510 (+7 ms)Region opened successfully at 1733497358511 (+1 ms) 2024-12-06T15:02:38,513 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7., pid=6, masterSystemTime=1733497358483 2024-12-06T15:02:38,517 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:02:38,517 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:02:38,518 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b49cda5460cc4f0dddbd13c60efce4f7, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:38,522 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b49cda5460cc4f0dddbd13c60efce4f7, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:02:38,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T15:02:38,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b49cda5460cc4f0dddbd13c60efce4f7, server=bb7ea3e46d9c,45829,1733497324439 in 196 msec 2024-12-06T15:02:38,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T15:02:38,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=b49cda5460cc4f0dddbd13c60efce4f7, ASSIGN in 366 msec 2024-12-06T15:02:38,534 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:02:38,534 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497358534"}]},"ts":"1733497358534"} 2024-12-06T15:02:38,538 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-06T15:02:38,539 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:02:38,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 845 msec 2024-12-06T15:02:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T15:02:38,722 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-06T15:02:38,722 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-06T15:02:38,725 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:02:38,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-06T15:02:38,731 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:02:38,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-06T15:02:38,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T15:02:38,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=-1] 2024-12-06T15:02:38,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T15:02:38,738 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T15:02:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:02:38,759 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:02:38,764 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-06T15:02:38,768 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-06T15:02:38,772 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-06T15:02:38,774 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-06T15:02:38,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=49553361271259144 2024-12-06T15:02:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497361773, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:02:38,798 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-06T15:02:38,799 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-06T15:02:38,801 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:02:38,802 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:02:38,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=49553361271259146 2024-12-06T15:02:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497361802, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:02:38,807 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:02:38,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=49553361271259147 2024-12-06T15:02:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497361806, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:02:38,811 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:02:38,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=49553361271259148 2024-12-06T15:02:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497361810, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:02:38,815 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:02:38,825 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_20886030_22 at /127.0.0.1:60902 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=6681 (was 6693) 2024-12-06T15:02:38,832 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=6681 2024-12-06T15:02:38,835 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:02:38,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-06T15:02:38,840 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:02:38,840 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:38,840 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-06T15:02:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T15:02:38,842 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:02:38,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741837_1013 (size=414) 2024-12-06T15:02:39,258 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e0189a0acac5efc8085840ccbd16c091, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:02:39,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741838_1014 (size=65) 2024-12-06T15:02:39,671 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:39,672 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing e0189a0acac5efc8085840ccbd16c091, disabling compactions & flushes 2024-12-06T15:02:39,672 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:02:39,672 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:02:39,672 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. after waiting 0 ms 2024-12-06T15:02:39,672 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:02:39,672 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:02:39,672 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for e0189a0acac5efc8085840ccbd16c091: Waiting for close lock at 1733497359671Disabling compacts and flushes for region at 1733497359671Disabling writes for close at 1733497359672 (+1 ms)Writing region close event to WAL at 1733497359672Closed at 1733497359672 2024-12-06T15:02:39,676 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:02:39,676 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733497359676"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497359676"}]},"ts":"1733497359676"} 2024-12-06T15:02:39,682 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:02:39,684 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:02:39,684 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497359684"}]},"ts":"1733497359684"} 2024-12-06T15:02:39,687 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-06T15:02:39,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e0189a0acac5efc8085840ccbd16c091, ASSIGN}] 2024-12-06T15:02:39,690 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e0189a0acac5efc8085840ccbd16c091, ASSIGN 2024-12-06T15:02:39,692 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e0189a0acac5efc8085840ccbd16c091, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:02:39,843 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e0189a0acac5efc8085840ccbd16c091, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:39,852 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e0189a0acac5efc8085840ccbd16c091, ASSIGN because future has completed 2024-12-06T15:02:39,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0189a0acac5efc8085840ccbd16c091, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:02:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T15:02:40,019 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:02:40,019 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e0189a0acac5efc8085840ccbd16c091, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:02:40,019 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,020 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:40,020 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,020 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,023 INFO [StoreOpener-e0189a0acac5efc8085840ccbd16c091-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,026 INFO [StoreOpener-e0189a0acac5efc8085840ccbd16c091-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0189a0acac5efc8085840ccbd16c091 columnFamilyName testFamily 2024-12-06T15:02:40,026 DEBUG [StoreOpener-e0189a0acac5efc8085840ccbd16c091-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:40,027 INFO [StoreOpener-e0189a0acac5efc8085840ccbd16c091-1 {}] regionserver.HStore(327): Store=e0189a0acac5efc8085840ccbd16c091/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:02:40,027 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,029 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,030 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,030 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,030 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,033 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,037 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:02:40,038 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened e0189a0acac5efc8085840ccbd16c091; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65557215, jitterRate=-0.023121371865272522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:02:40,038 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:02:40,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e0189a0acac5efc8085840ccbd16c091: Running coprocessor pre-open hook at 1733497360020Writing region info on filesystem at 1733497360020Initializing all the Stores at 1733497360022 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497360022Cleaning up temporary data from old regions at 1733497360030 (+8 ms)Running coprocessor post-open hooks at 1733497360038 (+8 ms)Region opened successfully at 1733497360039 (+1 ms) 2024-12-06T15:02:40,041 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., pid=9, masterSystemTime=1733497360008 2024-12-06T15:02:40,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:02:40,045 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:02:40,046 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e0189a0acac5efc8085840ccbd16c091, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:40,049 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0189a0acac5efc8085840ccbd16c091, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:02:40,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-06T15:02:40,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e0189a0acac5efc8085840ccbd16c091, server=bb7ea3e46d9c,45829,1733497324439 in 198 msec 2024-12-06T15:02:40,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T15:02:40,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e0189a0acac5efc8085840ccbd16c091, ASSIGN in 366 msec 2024-12-06T15:02:40,059 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:02:40,059 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497360059"}]},"ts":"1733497360059"} 2024-12-06T15:02:40,062 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-06T15:02:40,063 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:02:40,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2270 sec 2024-12-06T15:02:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-06T15:02:41,878 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-06T15:02:41,878 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-06T15:02:41,879 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:02:41,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-06T15:02:41,886 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:02:41,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-06T15:02:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:02:41,903 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:02:41,903 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-06T15:02:41,907 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-06T15:02:41,909 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-06T15:02:42,800 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:02:42,825 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-06T15:02:42,826 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T15:02:42,826 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-06T15:02:44,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T15:02:44,013 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-06T15:02:44,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T15:02:44,015 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-06T15:02:44,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-06T15:02:44,015 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-06T15:02:44,017 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-06T15:02:44,017 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-06T15:02:47,927 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=bb7ea3e46d9c:45829 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6017ms, rpcTimeout=5999ms 2024-12-06T15:02:47,928 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6017ms, rpcTimeout=5999ms 2024-12-06T15:02:47,928 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 from cache 2024-12-06T15:02:47,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=6669 (was 6681) 2024-12-06T15:02:47,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=6669 2024-12-06T15:02:47,946 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:02:47,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-06T15:02:47,950 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:02:47,950 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:47,950 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-06T15:02:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-06T15:02:47,952 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:02:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741839_1015 (size=412) 2024-12-06T15:02:48,366 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 690f6a1423f23a78c40cb2559261b5c4, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:02:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 87 bytes 2024-12-06T15:02:48,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741840_1016 (size=63) 2024-12-06T15:02:48,928 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:48,928 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 690f6a1423f23a78c40cb2559261b5c4, disabling compactions & flushes 2024-12-06T15:02:48,929 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:02:48,929 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:02:48,929 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. after waiting 0 ms 2024-12-06T15:02:48,929 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:02:48,929 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:02:48,929 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 690f6a1423f23a78c40cb2559261b5c4: Waiting for close lock at 1733497368928Disabling compacts and flushes for region at 1733497368928Disabling writes for close at 1733497368929 (+1 ms)Writing region close event to WAL at 1733497368929Closed at 1733497368929 2024-12-06T15:02:48,934 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:02:48,935 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733497368934"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497368934"}]},"ts":"1733497368934"} 2024-12-06T15:02:48,938 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:02:48,940 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:02:48,940 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497368940"}]},"ts":"1733497368940"} 2024-12-06T15:02:48,943 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-06T15:02:48,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=690f6a1423f23a78c40cb2559261b5c4, ASSIGN}] 2024-12-06T15:02:48,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=690f6a1423f23a78c40cb2559261b5c4, ASSIGN 2024-12-06T15:02:48,946 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=690f6a1423f23a78c40cb2559261b5c4, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:02:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-06T15:02:49,097 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=690f6a1423f23a78c40cb2559261b5c4, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:49,106 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=690f6a1423f23a78c40cb2559261b5c4, ASSIGN because future has completed 2024-12-06T15:02:49,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 690f6a1423f23a78c40cb2559261b5c4, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:02:49,272 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:02:49,273 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 690f6a1423f23a78c40cb2559261b5c4, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:02:49,273 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:49,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,277 INFO [StoreOpener-690f6a1423f23a78c40cb2559261b5c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,280 INFO [StoreOpener-690f6a1423f23a78c40cb2559261b5c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690f6a1423f23a78c40cb2559261b5c4 columnFamilyName testFamily 2024-12-06T15:02:49,280 DEBUG [StoreOpener-690f6a1423f23a78c40cb2559261b5c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:49,280 INFO [StoreOpener-690f6a1423f23a78c40cb2559261b5c4-1 {}] regionserver.HStore(327): Store=690f6a1423f23a78c40cb2559261b5c4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:02:49,281 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,282 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,283 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,283 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,283 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,286 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,289 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:02:49,290 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 690f6a1423f23a78c40cb2559261b5c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67523671, jitterRate=0.006181105971336365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:02:49,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:02:49,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 690f6a1423f23a78c40cb2559261b5c4: Running coprocessor pre-open hook at 1733497369274Writing region info on filesystem at 1733497369274Initializing all the Stores at 1733497369276 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497369276Cleaning up temporary data from old regions at 1733497369283 (+7 ms)Running coprocessor post-open hooks at 1733497369290 (+7 ms)Region opened successfully at 1733497369291 (+1 ms) 2024-12-06T15:02:49,292 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4., pid=12, masterSystemTime=1733497369261 2024-12-06T15:02:49,295 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:02:49,295 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:02:49,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=690f6a1423f23a78c40cb2559261b5c4, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:49,299 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 690f6a1423f23a78c40cb2559261b5c4, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:02:49,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-06T15:02:49,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 690f6a1423f23a78c40cb2559261b5c4, server=bb7ea3e46d9c,45829,1733497324439 in 195 msec 2024-12-06T15:02:49,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-06T15:02:49,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=690f6a1423f23a78c40cb2559261b5c4, ASSIGN in 361 msec 2024-12-06T15:02:49,310 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:02:49,311 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497369310"}]},"ts":"1733497369310"} 2024-12-06T15:02:49,314 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-06T15:02:49,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:02:49,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.3690 sec 2024-12-06T15:02:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-06T15:02:50,979 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-06T15:02:50,979 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-06T15:02:50,980 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:02:50,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-06T15:02:50,988 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:02:50,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-06T15:02:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:02:51,003 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:02:51,004 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-06T15:02:51,007 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-06T15:02:51,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-06T15:02:51,010 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-06T15:02:51,011 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-06T15:02:54,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-06T15:02:54,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-06T15:02:54,802 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:02:54,828 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-06T15:02:57,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45829 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259157' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-06T15:02:57,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45829 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 39 connection: 172.17.0.2:35434 deadline: 1733497383021, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259157' does not exist 2024-12-06T15:02:57,029 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to bb7ea3e46d9c,45829,1733497324439 for closing scanner id = 49553361271259157 for 690f6a1423f23a78c40cb2559261b5c4 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259157' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:02:57,033 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=6655 (was 6669) 2024-12-06T15:02:57,042 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=168, ProcessCount=11, AvailableMemoryMB=6655 2024-12-06T15:02:57,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:02:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-06T15:02:57,048 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:02:57,048 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:57,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-06T15:02:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T15:02:57,050 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:02:57,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741841_1017 (size=423) 2024-12-06T15:02:57,469 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 69351ee18399c7dd21c5688d68629c27, NAME => 'testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:02:57,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741842_1018 (size=74) 2024-12-06T15:02:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 81 bytes 2024-12-06T15:02:57,888 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:57,888 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 69351ee18399c7dd21c5688d68629c27, disabling compactions & flushes 2024-12-06T15:02:57,888 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:02:57,889 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:02:57,889 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. after waiting 0 ms 2024-12-06T15:02:57,889 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:02:57,889 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:02:57,889 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 69351ee18399c7dd21c5688d68629c27: Waiting for close lock at 1733497377888Disabling compacts and flushes for region at 1733497377888Disabling writes for close at 1733497377889 (+1 ms)Writing region close event to WAL at 1733497377889Closed at 1733497377889 2024-12-06T15:02:57,893 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:02:57,894 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733497377893"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497377893"}]},"ts":"1733497377893"} 2024-12-06T15:02:57,898 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:02:57,900 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:02:57,900 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497377900"}]},"ts":"1733497377900"} 2024-12-06T15:02:57,903 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-06T15:02:57,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=69351ee18399c7dd21c5688d68629c27, ASSIGN}] 2024-12-06T15:02:57,905 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=69351ee18399c7dd21c5688d68629c27, ASSIGN 2024-12-06T15:02:57,906 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=69351ee18399c7dd21c5688d68629c27, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:02:58,057 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=69351ee18399c7dd21c5688d68629c27, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:58,061 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=69351ee18399c7dd21c5688d68629c27, ASSIGN because future has completed 2024-12-06T15:02:58,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69351ee18399c7dd21c5688d68629c27, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:02:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T15:02:58,227 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:02:58,228 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 69351ee18399c7dd21c5688d68629c27, NAME => 'testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:02:58,228 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,228 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:02:58,229 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,229 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,231 INFO [StoreOpener-69351ee18399c7dd21c5688d68629c27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,235 INFO [StoreOpener-69351ee18399c7dd21c5688d68629c27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69351ee18399c7dd21c5688d68629c27 columnFamilyName testFamily 2024-12-06T15:02:58,236 DEBUG [StoreOpener-69351ee18399c7dd21c5688d68629c27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:02:58,237 INFO [StoreOpener-69351ee18399c7dd21c5688d68629c27-1 {}] regionserver.HStore(327): Store=69351ee18399c7dd21c5688d68629c27/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:02:58,237 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,238 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,239 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,240 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,240 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,243 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,247 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:02:58,248 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 69351ee18399c7dd21c5688d68629c27; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72836608, jitterRate=0.08535003662109375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:02:58,248 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:02:58,249 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 69351ee18399c7dd21c5688d68629c27: Running coprocessor pre-open hook at 1733497378229Writing region info on filesystem at 1733497378229Initializing all the Stores at 1733497378231 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497378231Cleaning up temporary data from old regions at 1733497378240 (+9 ms)Running coprocessor post-open hooks at 1733497378248 (+8 ms)Region opened successfully at 1733497378249 (+1 ms) 2024-12-06T15:02:58,251 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27., pid=15, masterSystemTime=1733497378216 2024-12-06T15:02:58,254 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:02:58,254 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:02:58,255 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=69351ee18399c7dd21c5688d68629c27, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:02:58,258 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69351ee18399c7dd21c5688d68629c27, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:02:58,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-06T15:02:58,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 69351ee18399c7dd21c5688d68629c27, server=bb7ea3e46d9c,45829,1733497324439 in 197 msec 2024-12-06T15:02:58,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-06T15:02:58,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=69351ee18399c7dd21c5688d68629c27, ASSIGN in 359 msec 2024-12-06T15:02:58,267 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:02:58,267 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497378267"}]},"ts":"1733497378267"} 2024-12-06T15:02:58,269 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-06T15:02:58,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:02:58,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2260 sec 2024-12-06T15:03:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-06T15:03:00,079 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-06T15:03:00,079 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-06T15:03:00,080 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:00,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-06T15:03:00,089 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:00,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-06T15:03:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:00,102 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:03:00,102 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-06T15:03:00,105 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-06T15:03:00,107 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-06T15:03:00,108 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-06T15:03:00,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=49553361271259161 2024-12-06T15:03:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497383107, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:03:00,112 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-06T15:03:00,113 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-06T15:03:00,114 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:03:00,116 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:03:00,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=49553361271259163 2024-12-06T15:03:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497383115, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:03:00,119 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:03:00,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=49553361271259164 2024-12-06T15:03:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497383119, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:03:00,124 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:03:00,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=49553361271259165 2024-12-06T15:03:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 42 connection: 172.17.0.2:35434 deadline: 1733497383123, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-06T15:03:00,128 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-06T15:03:00,137 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_20886030_22 at /127.0.0.1:53950 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=154 (was 168), ProcessCount=11 (was 11), AvailableMemoryMB=6652 (was 6655) 2024-12-06T15:03:00,147 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=154, ProcessCount=11, AvailableMemoryMB=6651 2024-12-06T15:03:00,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:03:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-06T15:03:00,154 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:03:00,154 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:00,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-06T15:03:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-06T15:03:00,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:03:00,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741843_1019 (size=418) 2024-12-06T15:03:00,571 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 250bfe9d03810615ad16441d6f2803b2, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:03:00,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741844_1020 (size=69) 2024-12-06T15:03:00,829 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-06T15:03:01,013 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:01,013 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 250bfe9d03810615ad16441d6f2803b2, disabling compactions & flushes 2024-12-06T15:03:01,013 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:01,014 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:01,014 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. after waiting 0 ms 2024-12-06T15:03:01,014 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:01,014 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:01,014 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 250bfe9d03810615ad16441d6f2803b2: Waiting for close lock at 1733497381013Disabling compacts and flushes for region at 1733497381013Disabling writes for close at 1733497381014 (+1 ms)Writing region close event to WAL at 1733497381014Closed at 1733497381014 2024-12-06T15:03:01,018 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:03:01,018 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733497381018"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497381018"}]},"ts":"1733497381018"} 2024-12-06T15:03:01,022 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:03:01,023 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:03:01,024 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497381023"}]},"ts":"1733497381023"} 2024-12-06T15:03:01,027 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-06T15:03:01,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=250bfe9d03810615ad16441d6f2803b2, ASSIGN}] 2024-12-06T15:03:01,029 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=250bfe9d03810615ad16441d6f2803b2, ASSIGN 2024-12-06T15:03:01,031 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=250bfe9d03810615ad16441d6f2803b2, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:03:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-06T15:03:01,182 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=250bfe9d03810615ad16441d6f2803b2, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:01,187 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=250bfe9d03810615ad16441d6f2803b2, ASSIGN because future has completed 2024-12-06T15:03:01,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 250bfe9d03810615ad16441d6f2803b2, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:03:01,354 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:01,354 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 250bfe9d03810615ad16441d6f2803b2, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:03:01,355 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,355 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:01,355 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,355 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,357 INFO [StoreOpener-250bfe9d03810615ad16441d6f2803b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,359 INFO [StoreOpener-250bfe9d03810615ad16441d6f2803b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 250bfe9d03810615ad16441d6f2803b2 columnFamilyName testFamily 2024-12-06T15:03:01,359 DEBUG [StoreOpener-250bfe9d03810615ad16441d6f2803b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:01,360 INFO [StoreOpener-250bfe9d03810615ad16441d6f2803b2-1 {}] regionserver.HStore(327): Store=250bfe9d03810615ad16441d6f2803b2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:03:01,360 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,361 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,362 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,363 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,363 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,365 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,368 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:03:01,369 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 250bfe9d03810615ad16441d6f2803b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59417506, jitterRate=-0.11461016535758972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:03:01,369 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:01,370 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 250bfe9d03810615ad16441d6f2803b2: Running coprocessor pre-open hook at 1733497381355Writing region info on filesystem at 1733497381355Initializing all the Stores at 1733497381357 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497381357Cleaning up temporary data from old regions at 1733497381363 (+6 ms)Running coprocessor post-open hooks at 1733497381369 (+6 ms)Region opened successfully at 1733497381370 (+1 ms) 2024-12-06T15:03:01,371 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., pid=18, masterSystemTime=1733497381342 2024-12-06T15:03:01,373 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:01,374 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:01,374 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=250bfe9d03810615ad16441d6f2803b2, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:01,377 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 250bfe9d03810615ad16441d6f2803b2, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:03:01,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-06T15:03:01,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 250bfe9d03810615ad16441d6f2803b2, server=bb7ea3e46d9c,45829,1733497324439 in 191 msec 2024-12-06T15:03:01,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-06T15:03:01,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=250bfe9d03810615ad16441d6f2803b2, ASSIGN in 354 msec 2024-12-06T15:03:01,386 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:03:01,386 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497381386"}]},"ts":"1733497381386"} 2024-12-06T15:03:01,389 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-06T15:03:01,390 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:03:01,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2400 sec 2024-12-06T15:03:02,691 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T15:03:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-06T15:03:03,187 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-06T15:03:03,187 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-06T15:03:03,188 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:03,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-06T15:03:03,194 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:03,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-06T15:03:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:03,211 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:03:03,211 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-06T15:03:03,214 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-06T15:03:03,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-06T15:03:03,732 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:03:04,014 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-06T15:03:04,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-06T15:03:04,016 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-06T15:03:04,016 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-06T15:03:04,217 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=bb7ea3e46d9c:45829 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-12-06T15:03:04,217 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-12-06T15:03:04,217 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 from cache 2024-12-06T15:03:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 87 bytes 2024-12-06T15:03:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:05,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-06T15:03:06,216 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=bb7ea3e46d9c:45829 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=973ms, rpcTimeout=971ms 2024-12-06T15:03:06,217 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=973ms, rpcTimeout=971ms 2024-12-06T15:03:06,217 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 from cache 2024-12-06T15:03:06,232 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=150 (was 154), ProcessCount=11 (was 11), AvailableMemoryMB=6643 (was 6651) 2024-12-06T15:03:06,244 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=6642 2024-12-06T15:03:06,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:03:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-06T15:03:06,252 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:03:06,252 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:06,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-06T15:03:06,254 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:03:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T15:03:06,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741845_1021 (size=416) 2024-12-06T15:03:06,669 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f23b08a30d88172abcf16915983abac9, NAME => 'testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:03:06,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741846_1022 (size=67) 2024-12-06T15:03:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 87 bytes 2024-12-06T15:03:06,832 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-06T15:03:07,087 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:07,088 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing f23b08a30d88172abcf16915983abac9, disabling compactions & flushes 2024-12-06T15:03:07,088 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:07,088 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:07,088 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. after waiting 0 ms 2024-12-06T15:03:07,088 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:07,089 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:07,089 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f23b08a30d88172abcf16915983abac9: Waiting for close lock at 1733497387087Disabling compacts and flushes for region at 1733497387087Disabling writes for close at 1733497387088 (+1 ms)Writing region close event to WAL at 1733497387088Closed at 1733497387088 2024-12-06T15:03:07,092 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:03:07,092 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733497387092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497387092"}]},"ts":"1733497387092"} 2024-12-06T15:03:07,095 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:03:07,096 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:03:07,096 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497387096"}]},"ts":"1733497387096"} 2024-12-06T15:03:07,099 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-06T15:03:07,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f23b08a30d88172abcf16915983abac9, ASSIGN}] 2024-12-06T15:03:07,100 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f23b08a30d88172abcf16915983abac9, ASSIGN 2024-12-06T15:03:07,102 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f23b08a30d88172abcf16915983abac9, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:03:07,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f23b08a30d88172abcf16915983abac9, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:07,260 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f23b08a30d88172abcf16915983abac9, ASSIGN because future has completed 2024-12-06T15:03:07,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f23b08a30d88172abcf16915983abac9, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:03:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T15:03:07,423 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:07,423 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => f23b08a30d88172abcf16915983abac9, NAME => 'testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:03:07,424 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,424 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:07,424 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,425 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,427 INFO [StoreOpener-f23b08a30d88172abcf16915983abac9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,430 INFO [StoreOpener-f23b08a30d88172abcf16915983abac9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f23b08a30d88172abcf16915983abac9 columnFamilyName testFamily 2024-12-06T15:03:07,430 DEBUG [StoreOpener-f23b08a30d88172abcf16915983abac9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:07,431 INFO [StoreOpener-f23b08a30d88172abcf16915983abac9-1 {}] regionserver.HStore(327): Store=f23b08a30d88172abcf16915983abac9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:03:07,431 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,433 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,433 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,434 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,434 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,436 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,439 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:03:07,440 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened f23b08a30d88172abcf16915983abac9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75455651, jitterRate=0.1243768185377121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:03:07,440 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:07,441 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for f23b08a30d88172abcf16915983abac9: Running coprocessor pre-open hook at 1733497387425Writing region info on filesystem at 1733497387425Initializing all the Stores at 1733497387427 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497387427Cleaning up temporary data from old regions at 1733497387434 (+7 ms)Running coprocessor post-open hooks at 1733497387440 (+6 ms)Region opened successfully at 1733497387441 (+1 ms) 2024-12-06T15:03:07,442 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9., pid=21, masterSystemTime=1733497387417 2024-12-06T15:03:07,445 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:07,445 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:07,446 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f23b08a30d88172abcf16915983abac9, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:07,449 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f23b08a30d88172abcf16915983abac9, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:03:07,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-06T15:03:07,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure f23b08a30d88172abcf16915983abac9, server=bb7ea3e46d9c,45829,1733497324439 in 189 msec 2024-12-06T15:03:07,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-06T15:03:07,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f23b08a30d88172abcf16915983abac9, ASSIGN in 354 msec 2024-12-06T15:03:07,456 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:03:07,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497387456"}]},"ts":"1733497387456"} 2024-12-06T15:03:07,459 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-06T15:03:07,460 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:03:07,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2130 sec 2024-12-06T15:03:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-06T15:03:09,277 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-06T15:03:09,277 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-06T15:03:09,278 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:09,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-06T15:03:09,287 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:09,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-06T15:03:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:09,300 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:03:09,300 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-06T15:03:09,303 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-06T15:03:09,305 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-06T15:03:09,306 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-06T15:03:09,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-06T15:03:12,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45829 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259176' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-06T15:03:12,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45829 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 39 connection: 172.17.0.2:35434 deadline: 1733497393322, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259176' does not exist 2024-12-06T15:03:12,324 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to bb7ea3e46d9c,45829,1733497324439 for closing scanner id = 49553361271259176 for f23b08a30d88172abcf16915983abac9 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259176' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:03:12,329 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=6650 (was 6642) - AvailableMemoryMB LEAK? - 2024-12-06T15:03:12,338 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=6650 2024-12-06T15:03:12,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:03:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-06T15:03:12,344 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:03:12,344 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:12,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-06T15:03:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-06T15:03:12,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:03:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741847_1023 (size=411) 2024-12-06T15:03:12,761 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9b103a8da26efa416379a459c26ee2ab, NAME => 'testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:03:12,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741848_1024 (size=62) 2024-12-06T15:03:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 81 bytes 2024-12-06T15:03:12,927 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:03:12,946 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-06T15:03:13,174 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:13,175 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 9b103a8da26efa416379a459c26ee2ab, disabling compactions & flushes 2024-12-06T15:03:13,175 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:13,175 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:13,175 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. after waiting 0 ms 2024-12-06T15:03:13,175 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:13,175 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:13,175 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9b103a8da26efa416379a459c26ee2ab: Waiting for close lock at 1733497393175Disabling compacts and flushes for region at 1733497393175Disabling writes for close at 1733497393175Writing region close event to WAL at 1733497393175Closed at 1733497393175 2024-12-06T15:03:13,178 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:03:13,179 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733497393179"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497393179"}]},"ts":"1733497393179"} 2024-12-06T15:03:13,184 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:03:13,186 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:03:13,187 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497393187"}]},"ts":"1733497393187"} 2024-12-06T15:03:13,190 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-06T15:03:13,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=9b103a8da26efa416379a459c26ee2ab, ASSIGN}] 2024-12-06T15:03:13,193 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=9b103a8da26efa416379a459c26ee2ab, ASSIGN 2024-12-06T15:03:13,194 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=9b103a8da26efa416379a459c26ee2ab, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:03:13,346 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=9b103a8da26efa416379a459c26ee2ab, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:13,353 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=9b103a8da26efa416379a459c26ee2ab, ASSIGN because future has completed 2024-12-06T15:03:13,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b103a8da26efa416379a459c26ee2ab, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:03:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-06T15:03:13,521 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:13,522 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b103a8da26efa416379a459c26ee2ab, NAME => 'testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:03:13,522 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,522 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:13,522 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,522 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,525 INFO [StoreOpener-9b103a8da26efa416379a459c26ee2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,527 INFO [StoreOpener-9b103a8da26efa416379a459c26ee2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b103a8da26efa416379a459c26ee2ab columnFamilyName testFamily 2024-12-06T15:03:13,527 DEBUG [StoreOpener-9b103a8da26efa416379a459c26ee2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:13,528 INFO [StoreOpener-9b103a8da26efa416379a459c26ee2ab-1 {}] regionserver.HStore(327): Store=9b103a8da26efa416379a459c26ee2ab/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:03:13,528 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,529 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,530 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,531 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,531 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,534 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,539 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:03:13,540 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 9b103a8da26efa416379a459c26ee2ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68894703, jitterRate=0.026611074805259705}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:03:13,540 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:13,540 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 9b103a8da26efa416379a459c26ee2ab: Running coprocessor pre-open hook at 1733497393522Writing region info on filesystem at 1733497393522Initializing all the Stores at 1733497393524 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497393524Cleaning up temporary data from old regions at 1733497393531 (+7 ms)Running coprocessor post-open hooks at 1733497393540 (+9 ms)Region opened successfully at 1733497393540 2024-12-06T15:03:13,542 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab., pid=24, masterSystemTime=1733497393509 2024-12-06T15:03:13,545 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:13,545 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:13,546 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=9b103a8da26efa416379a459c26ee2ab, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:13,549 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b103a8da26efa416379a459c26ee2ab, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:03:13,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-06T15:03:13,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b103a8da26efa416379a459c26ee2ab, server=bb7ea3e46d9c,45829,1733497324439 in 197 msec 2024-12-06T15:03:13,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-06T15:03:13,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=9b103a8da26efa416379a459c26ee2ab, ASSIGN in 363 msec 2024-12-06T15:03:13,558 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:03:13,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497393558"}]},"ts":"1733497393558"} 2024-12-06T15:03:13,561 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-06T15:03:13,562 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:03:13,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2220 sec 2024-12-06T15:03:14,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-06T15:03:14,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-06T15:03:14,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-06T15:03:14,015 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-06T15:03:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-06T15:03:15,366 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-06T15:03:15,366 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-06T15:03:15,367 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:15,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-06T15:03:15,377 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:15,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-06T15:03:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:15,388 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:03:15,388 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-06T15:03:15,391 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-06T15:03:15,393 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-06T15:03:15,394 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-06T15:03:15,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-06T15:03:18,198 INFO [master/bb7ea3e46d9c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-06T15:03:18,198 INFO [master/bb7ea3e46d9c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-06T15:03:18,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45829 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259180' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-06T15:03:18,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45829 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 39 connection: 172.17.0.2:35434 deadline: 1733497399404, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259180' does not exist 2024-12-06T15:03:18,405 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to bb7ea3e46d9c,45829,1733497324439 for closing scanner id = 49553361271259180 for 9b103a8da26efa416379a459c26ee2ab of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259180' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:03:18,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=6644 (was 6650) 2024-12-06T15:03:18,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=6644 2024-12-06T15:03:18,419 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:03:18,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-06T15:03:18,422 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:03:18,422 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:18,422 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-06T15:03:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T15:03:18,424 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:03:18,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741849_1025 (size=419) 2024-12-06T15:03:18,840 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8ce8bdc7344980bd3a5582a41147924, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:03:18,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741850_1026 (size=70) 2024-12-06T15:03:18,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 81 bytes 2024-12-06T15:03:19,026 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:03:19,056 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-06T15:03:19,249 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:19,249 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing f8ce8bdc7344980bd3a5582a41147924, disabling compactions & flushes 2024-12-06T15:03:19,249 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:19,249 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:19,249 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. after waiting 0 ms 2024-12-06T15:03:19,250 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:19,250 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:19,250 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8ce8bdc7344980bd3a5582a41147924: Waiting for close lock at 1733497399249Disabling compacts and flushes for region at 1733497399249Disabling writes for close at 1733497399249Writing region close event to WAL at 1733497399250 (+1 ms)Closed at 1733497399250 2024-12-06T15:03:19,253 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:03:19,253 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733497399253"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497399253"}]},"ts":"1733497399253"} 2024-12-06T15:03:19,256 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:03:19,257 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:03:19,257 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497399257"}]},"ts":"1733497399257"} 2024-12-06T15:03:19,260 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-06T15:03:19,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f8ce8bdc7344980bd3a5582a41147924, ASSIGN}] 2024-12-06T15:03:19,262 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f8ce8bdc7344980bd3a5582a41147924, ASSIGN 2024-12-06T15:03:19,263 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f8ce8bdc7344980bd3a5582a41147924, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:03:19,414 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f8ce8bdc7344980bd3a5582a41147924, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:19,420 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f8ce8bdc7344980bd3a5582a41147924, ASSIGN because future has completed 2024-12-06T15:03:19,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8ce8bdc7344980bd3a5582a41147924, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:03:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T15:03:19,588 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:19,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => f8ce8bdc7344980bd3a5582a41147924, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:03:19,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:19,590 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,590 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,593 INFO [StoreOpener-f8ce8bdc7344980bd3a5582a41147924-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,596 INFO [StoreOpener-f8ce8bdc7344980bd3a5582a41147924-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8ce8bdc7344980bd3a5582a41147924 columnFamilyName testFamily 2024-12-06T15:03:19,596 DEBUG [StoreOpener-f8ce8bdc7344980bd3a5582a41147924-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:19,597 INFO [StoreOpener-f8ce8bdc7344980bd3a5582a41147924-1 {}] regionserver.HStore(327): Store=f8ce8bdc7344980bd3a5582a41147924/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:03:19,597 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,598 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,598 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,599 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,600 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,602 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,605 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:03:19,606 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened f8ce8bdc7344980bd3a5582a41147924; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63819227, jitterRate=-0.04901941120624542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:03:19,606 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:19,606 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for f8ce8bdc7344980bd3a5582a41147924: Running coprocessor pre-open hook at 1733497399590Writing region info on filesystem at 1733497399590Initializing all the Stores at 1733497399593 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497399593Cleaning up temporary data from old regions at 1733497399600 (+7 ms)Running coprocessor post-open hooks at 1733497399606 (+6 ms)Region opened successfully at 1733497399606 2024-12-06T15:03:19,608 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., pid=27, masterSystemTime=1733497399577 2024-12-06T15:03:19,610 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:19,610 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:19,611 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f8ce8bdc7344980bd3a5582a41147924, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:19,613 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8ce8bdc7344980bd3a5582a41147924, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:03:19,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-06T15:03:19,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure f8ce8bdc7344980bd3a5582a41147924, server=bb7ea3e46d9c,45829,1733497324439 in 194 msec 2024-12-06T15:03:19,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-06T15:03:19,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=f8ce8bdc7344980bd3a5582a41147924, ASSIGN in 357 msec 2024-12-06T15:03:19,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:03:19,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497399621"}]},"ts":"1733497399621"} 2024-12-06T15:03:19,623 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-06T15:03:19,624 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:03:19,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2060 sec 2024-12-06T15:03:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-06T15:03:21,448 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-06T15:03:21,448 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-06T15:03:21,448 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:21,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-06T15:03:21,457 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:21,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-06T15:03:21,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:21,471 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:03:21,471 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-06T15:03:21,474 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-06T15:03:21,475 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-06T15:03:24,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-06T15:03:24,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-06T15:03:25,094 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:03:25,120 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-06T15:03:27,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=bb7ea3e46d9c:45829 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-06T15:03:27,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-06T15:03:27,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 from cache 2024-12-06T15:03:27,491 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=107 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=6640 (was 6644) 2024-12-06T15:03:27,501 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=107, ProcessCount=11, AvailableMemoryMB=6640 2024-12-06T15:03:27,504 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:03:27,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-06T15:03:27,507 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:03:27,507 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:27,508 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-06T15:03:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-06T15:03:27,509 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:03:27,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741851_1027 (size=407) 2024-12-06T15:03:27,921 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0fa8eedb71b823cffb3dcfaededd9e04, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:03:27,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741852_1028 (size=58) 2024-12-06T15:03:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 87 bytes 2024-12-06T15:03:28,335 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:28,336 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 0fa8eedb71b823cffb3dcfaededd9e04, disabling compactions & flushes 2024-12-06T15:03:28,336 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:28,336 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:28,336 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. after waiting 0 ms 2024-12-06T15:03:28,336 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:28,336 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:28,336 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0fa8eedb71b823cffb3dcfaededd9e04: Waiting for close lock at 1733497408336Disabling compacts and flushes for region at 1733497408336Disabling writes for close at 1733497408336Writing region close event to WAL at 1733497408336Closed at 1733497408336 2024-12-06T15:03:28,339 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:03:28,339 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733497408339"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497408339"}]},"ts":"1733497408339"} 2024-12-06T15:03:28,343 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:03:28,344 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:03:28,344 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497408344"}]},"ts":"1733497408344"} 2024-12-06T15:03:28,347 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-06T15:03:28,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0fa8eedb71b823cffb3dcfaededd9e04, ASSIGN}] 2024-12-06T15:03:28,348 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0fa8eedb71b823cffb3dcfaededd9e04, ASSIGN 2024-12-06T15:03:28,349 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0fa8eedb71b823cffb3dcfaededd9e04, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:03:28,501 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0fa8eedb71b823cffb3dcfaededd9e04, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:28,506 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0fa8eedb71b823cffb3dcfaededd9e04, ASSIGN because future has completed 2024-12-06T15:03:28,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa8eedb71b823cffb3dcfaededd9e04, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:03:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-06T15:03:28,668 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:28,669 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fa8eedb71b823cffb3dcfaededd9e04, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:03:28,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:28,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,673 INFO [StoreOpener-0fa8eedb71b823cffb3dcfaededd9e04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,675 INFO [StoreOpener-0fa8eedb71b823cffb3dcfaededd9e04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa8eedb71b823cffb3dcfaededd9e04 columnFamilyName testFamily 2024-12-06T15:03:28,675 DEBUG [StoreOpener-0fa8eedb71b823cffb3dcfaededd9e04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:28,676 INFO [StoreOpener-0fa8eedb71b823cffb3dcfaededd9e04-1 {}] regionserver.HStore(327): Store=0fa8eedb71b823cffb3dcfaededd9e04/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:03:28,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,677 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,677 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,678 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,678 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,683 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:03:28,684 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 0fa8eedb71b823cffb3dcfaededd9e04; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74441671, jitterRate=0.10926733911037445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:03:28,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:28,685 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 0fa8eedb71b823cffb3dcfaededd9e04: Running coprocessor pre-open hook at 1733497408670Writing region info on filesystem at 1733497408670Initializing all the Stores at 1733497408672 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497408672Cleaning up temporary data from old regions at 1733497408678 (+6 ms)Running coprocessor post-open hooks at 1733497408684 (+6 ms)Region opened successfully at 1733497408685 (+1 ms) 2024-12-06T15:03:28,686 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04., pid=30, masterSystemTime=1733497408662 2024-12-06T15:03:28,688 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:28,688 INFO [RS_OPEN_PRIORITY_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:28,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0fa8eedb71b823cffb3dcfaededd9e04, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:28,691 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fa8eedb71b823cffb3dcfaededd9e04, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:03:28,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-06T15:03:28,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fa8eedb71b823cffb3dcfaededd9e04, server=bb7ea3e46d9c,45829,1733497324439 in 186 msec 2024-12-06T15:03:28,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-06T15:03:28,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0fa8eedb71b823cffb3dcfaededd9e04, ASSIGN in 348 msec 2024-12-06T15:03:28,698 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:03:28,698 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497408698"}]},"ts":"1733497408698"} 2024-12-06T15:03:28,700 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-06T15:03:28,701 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:03:28,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1970 sec 2024-12-06T15:03:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-06T15:03:30,538 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-06T15:03:30,538 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-06T15:03:30,539 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:30,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-06T15:03:30,551 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:30,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-06T15:03:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:30,563 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:03:30,563 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-06T15:03:30,566 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-06T15:03:30,567 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-06T15:03:30,568 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-06T15:03:30,569 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-06T15:03:31,122 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-06T15:03:32,692 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T15:03:34,174 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:03:34,212 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-06T15:03:34,213 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-06T15:03:36,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45829 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259188' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-06T15:03:36,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45829 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 39 connection: 172.17.0.2:35434 deadline: 1733497422578, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259188' does not exist 2024-12-06T15:03:36,580 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to bb7ea3e46d9c,45829,1733497324439 for closing scanner id = 49553361271259188 for 0fa8eedb71b823cffb3dcfaededd9e04 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '49553361271259188' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T15:03:36,583 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=91 (was 107), ProcessCount=11 (was 11), AvailableMemoryMB=6631 (was 6640) 2024-12-06T15:03:36,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=91, ProcessCount=11, AvailableMemoryMB=6630 2024-12-06T15:03:36,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T15:03:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-06T15:03:36,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T15:03:36,596 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:36,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-06T15:03:36,597 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T15:03:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-06T15:03:36,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741853_1029 (size=423) 2024-12-06T15:03:37,011 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29c7794be9ee9097a0bcfd6f62f3fe79, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf 2024-12-06T15:03:37,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741854_1030 (size=74) 2024-12-06T15:03:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 81 bytes 2024-12-06T15:03:37,427 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:37,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 29c7794be9ee9097a0bcfd6f62f3fe79, disabling compactions & flushes 2024-12-06T15:03:37,428 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:37,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:37,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. after waiting 0 ms 2024-12-06T15:03:37,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:37,428 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:37,428 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29c7794be9ee9097a0bcfd6f62f3fe79: Waiting for close lock at 1733497417427Disabling compacts and flushes for region at 1733497417427Disabling writes for close at 1733497417428 (+1 ms)Writing region close event to WAL at 1733497417428Closed at 1733497417428 2024-12-06T15:03:37,431 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T15:03:37,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733497417432"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733497417432"}]},"ts":"1733497417432"} 2024-12-06T15:03:37,436 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T15:03:37,438 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T15:03:37,438 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497417438"}]},"ts":"1733497417438"} 2024-12-06T15:03:37,441 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-06T15:03:37,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=29c7794be9ee9097a0bcfd6f62f3fe79, ASSIGN}] 2024-12-06T15:03:37,444 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=29c7794be9ee9097a0bcfd6f62f3fe79, ASSIGN 2024-12-06T15:03:37,445 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=29c7794be9ee9097a0bcfd6f62f3fe79, ASSIGN; state=OFFLINE, location=bb7ea3e46d9c,45829,1733497324439; forceNewPlan=false, retain=false 2024-12-06T15:03:37,504 DEBUG [master/bb7ea3e46d9c:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T15:03:37,596 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=29c7794be9ee9097a0bcfd6f62f3fe79, regionState=OPENING, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:37,599 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=29c7794be9ee9097a0bcfd6f62f3fe79, ASSIGN because future has completed 2024-12-06T15:03:37,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29c7794be9ee9097a0bcfd6f62f3fe79, server=bb7ea3e46d9c,45829,1733497324439}] 2024-12-06T15:03:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-06T15:03:37,757 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:37,757 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 29c7794be9ee9097a0bcfd6f62f3fe79, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79.', STARTKEY => '', ENDKEY => ''} 2024-12-06T15:03:37,757 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,757 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T15:03:37,758 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,758 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,760 INFO [StoreOpener-29c7794be9ee9097a0bcfd6f62f3fe79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,763 INFO [StoreOpener-29c7794be9ee9097a0bcfd6f62f3fe79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29c7794be9ee9097a0bcfd6f62f3fe79 columnFamilyName testFamily 2024-12-06T15:03:37,764 DEBUG [StoreOpener-29c7794be9ee9097a0bcfd6f62f3fe79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T15:03:37,765 INFO [StoreOpener-29c7794be9ee9097a0bcfd6f62f3fe79-1 {}] regionserver.HStore(327): Store=29c7794be9ee9097a0bcfd6f62f3fe79/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T15:03:37,765 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,767 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,768 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,769 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,769 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,773 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,778 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T15:03:37,779 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 29c7794be9ee9097a0bcfd6f62f3fe79; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61850534, jitterRate=-0.07835522294044495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T15:03:37,779 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:37,780 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 29c7794be9ee9097a0bcfd6f62f3fe79: Running coprocessor pre-open hook at 1733497417758Writing region info on filesystem at 1733497417758Initializing all the Stores at 1733497417759 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733497417759Cleaning up temporary data from old regions at 1733497417769 (+10 ms)Running coprocessor post-open hooks at 1733497417779 (+10 ms)Region opened successfully at 1733497417780 (+1 ms) 2024-12-06T15:03:37,782 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., pid=33, masterSystemTime=1733497417751 2024-12-06T15:03:37,785 DEBUG [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:37,785 INFO [RS_OPEN_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:37,786 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=29c7794be9ee9097a0bcfd6f62f3fe79, regionState=OPEN, openSeqNum=2, regionLocation=bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:37,789 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29c7794be9ee9097a0bcfd6f62f3fe79, server=bb7ea3e46d9c,45829,1733497324439 because future has completed 2024-12-06T15:03:37,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-06T15:03:37,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 29c7794be9ee9097a0bcfd6f62f3fe79, server=bb7ea3e46d9c,45829,1733497324439 in 192 msec 2024-12-06T15:03:37,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-06T15:03:37,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=29c7794be9ee9097a0bcfd6f62f3fe79, ASSIGN in 351 msec 2024-12-06T15:03:37,797 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T15:03:37,797 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733497417797"}]},"ts":"1733497417797"} 2024-12-06T15:03:37,799 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-06T15:03:37,800 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T15:03:37,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2070 sec 2024-12-06T15:03:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-06T15:03:39,627 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-06T15:03:39,627 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-06T15:03:39,627 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:39,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-06T15:03:39,634 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T15:03:39,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-06T15:03:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:39,650 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-06T15:03:39,650 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-06T15:03:39,653 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-06T15:03:39,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-06T15:03:40,656 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=bb7ea3e46d9c:45829 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-12-06T15:03:40,656 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-12-06T15:03:40,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 from cache 2024-12-06T15:03:41,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 87 bytes 2024-12-06T15:03:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2] 2024-12-06T15:03:41,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45829 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-06T15:03:42,656 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=bb7ea3e46d9c:45829 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=988ms, rpcTimeout=985ms 2024-12-06T15:03:42,656 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=988ms, rpcTimeout=985ms 2024-12-06T15:03:42,656 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., hostname=bb7ea3e46d9c,45829,1733497324439, seqNum=2 from cache 2024-12-06T15:03:42,666 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=91 (was 91), ProcessCount=11 (was 11), AvailableMemoryMB=7280 (was 6630) - AvailableMemoryMB LEAK? - 2024-12-06T15:03:42,667 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T15:03:42,667 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T15:03:42,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:03:42,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:03:42,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T15:03:42,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T15:03:42,673 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T15:03:42,674 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T15:03:42,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:03:42,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:03:42,674 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T15:03:42,674 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T15:03:42,674 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1871265054, stopped=false 2024-12-06T15:03:42,675 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bb7ea3e46d9c,43691,1733497323635 2024-12-06T15:03:42,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T15:03:42,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:03:42,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T15:03:42,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T15:03:42,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:03:42,724 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T15:03:42,724 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T15:03:42,724 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T15:03:42,724 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T15:03:42,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:03:42,725 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bb7ea3e46d9c,45829,1733497324439' ***** 2024-12-06T15:03:42,725 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T15:03:42,725 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T15:03:42,725 INFO [RS:0;bb7ea3e46d9c:45829 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T15:03:42,725 INFO [RS:0;bb7ea3e46d9c:45829 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T15:03:42,725 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T15:03:42,726 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b103a8da26efa416379a459c26ee2ab 2024-12-06T15:03:42,728 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for 0fa8eedb71b823cffb3dcfaededd9e04 2024-12-06T15:03:42,728 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for f23b08a30d88172abcf16915983abac9 2024-12-06T15:03:42,728 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for b49cda5460cc4f0dddbd13c60efce4f7 2024-12-06T15:03:42,728 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for 690f6a1423f23a78c40cb2559261b5c4 2024-12-06T15:03:42,729 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b103a8da26efa416379a459c26ee2ab, disabling compactions & flushes 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for 250bfe9d03810615ad16441d6f2803b2 2024-12-06T15:03:42,729 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for e0189a0acac5efc8085840ccbd16c091 2024-12-06T15:03:42,729 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:42,729 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. after waiting 0 ms 2024-12-06T15:03:42,729 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for 29c7794be9ee9097a0bcfd6f62f3fe79 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(959): stopping server bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T15:03:42,729 INFO [RS:0;bb7ea3e46d9c:45829 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bb7ea3e46d9c:45829. 2024-12-06T15:03:42,729 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T15:03:42,729 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:03:42,730 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T15:03:42,730 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T15:03:42,730 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T15:03:42,730 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T15:03:42,730 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-06T15:03:42,730 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T15:03:42,731 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T15:03:42,731 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T15:03:42,731 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T15:03:42,731 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T15:03:42,731 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 0fa8eedb71b823cffb3dcfaededd9e04, 1588230740, 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 690f6a1423f23a78c40cb2559261b5c4, 69351ee18399c7dd21c5688d68629c27, 9b103a8da26efa416379a459c26ee2ab, b49cda5460cc4f0dddbd13c60efce4f7, e0189a0acac5efc8085840ccbd16c091, f23b08a30d88172abcf16915983abac9, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:42,733 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9b103a8da26efa416379a459c26ee2ab 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:42,733 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-06T15:03:42,737 INFO [regionserver/bb7ea3e46d9c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-06T15:03:42,737 INFO [regionserver/bb7ea3e46d9c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-06T15:03:42,808 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab/.tmp/testFamily/d42f23341f5048f497f6d963fd73ba25 is 61, key is row-0/testFamily:testQualifier/1733497395380/Put/seqid=0 2024-12-06T15:03:42,816 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/info/e5c1e18fb0f5482aaa78a7cbe15faf58 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7./info:regioninfo/1733497358518/Put/seqid=0 2024-12-06T15:03:42,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741856_1032 (size=5049) 2024-12-06T15:03:42,838 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab/.tmp/testFamily/d42f23341f5048f497f6d963fd73ba25 2024-12-06T15:03:42,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741855_1031 (size=20564) 2024-12-06T15:03:42,861 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/info/e5c1e18fb0f5482aaa78a7cbe15faf58 2024-12-06T15:03:42,931 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 0fa8eedb71b823cffb3dcfaededd9e04, 1588230740, 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 690f6a1423f23a78c40cb2559261b5c4, 69351ee18399c7dd21c5688d68629c27, 9b103a8da26efa416379a459c26ee2ab, b49cda5460cc4f0dddbd13c60efce4f7, e0189a0acac5efc8085840ccbd16c091, f23b08a30d88172abcf16915983abac9, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:42,940 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab/.tmp/testFamily/d42f23341f5048f497f6d963fd73ba25 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab/testFamily/d42f23341f5048f497f6d963fd73ba25 2024-12-06T15:03:42,955 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab/testFamily/d42f23341f5048f497f6d963fd73ba25, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:42,968 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9b103a8da26efa416379a459c26ee2ab in 233ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:42,981 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/ns/71de9e6ea4904cc391fd4a9161a89d72 is 43, key is default/ns:d/1733497357457/Put/seqid=0 2024-12-06T15:03:43,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741857_1033 (size=5153) 2024-12-06T15:03:43,016 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/ns/71de9e6ea4904cc391fd4a9161a89d72 2024-12-06T15:03:43,023 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNext/9b103a8da26efa416379a459c26ee2ab/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:43,030 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:43,030 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b103a8da26efa416379a459c26ee2ab: Waiting for close lock at 1733497422728Running coprocessor pre-close hooks at 1733497422728Disabling compacts and flushes for region at 1733497422728Disabling writes for close at 1733497422729 (+1 ms)Obtaining lock to block concurrent updates at 1733497422733 (+4 ms)Preparing flush snapshotting stores in 9b103a8da26efa416379a459c26ee2ab at 1733497422733Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497422742 (+9 ms)Flushing stores of testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. at 1733497422743 (+1 ms)Flushing 9b103a8da26efa416379a459c26ee2ab/testFamily: creating writer at 1733497422745 (+2 ms)Flushing 9b103a8da26efa416379a459c26ee2ab/testFamily: appending metadata at 1733497422802 (+57 ms)Flushing 9b103a8da26efa416379a459c26ee2ab/testFamily: closing flushed file at 1733497422805 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15b81e71: reopening flushed file at 1733497422936 (+131 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9b103a8da26efa416379a459c26ee2ab in 233ms, sequenceid=8, compaction requested=false at 1733497422968 (+32 ms)Writing region close event to WAL at 1733497423011 (+43 ms)Running coprocessor post-close hooks at 1733497423024 (+13 ms)Closed at 1733497423030 (+6 ms) 2024-12-06T15:03:43,031 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733497392340.9b103a8da26efa416379a459c26ee2ab. 2024-12-06T15:03:43,031 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0fa8eedb71b823cffb3dcfaededd9e04, disabling compactions & flushes 2024-12-06T15:03:43,031 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:43,031 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:43,031 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. after waiting 0 ms 2024-12-06T15:03:43,031 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:43,031 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0fa8eedb71b823cffb3dcfaededd9e04 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:43,065 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04/.tmp/testFamily/82f4b60f637f41b0b899993c8b02dae6 is 61, key is row-0/testFamily:testQualifier/1733497410555/Put/seqid=0 2024-12-06T15:03:43,079 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/table/ec47568bbbb9432c9160940c43bacf1c is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733497358534/Put/seqid=0 2024-12-06T15:03:43,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741858_1034 (size=5049) 2024-12-06T15:03:43,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741859_1035 (size=6699) 2024-12-06T15:03:43,105 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/table/ec47568bbbb9432c9160940c43bacf1c 2024-12-06T15:03:43,123 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/info/e5c1e18fb0f5482aaa78a7cbe15faf58 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/info/e5c1e18fb0f5482aaa78a7cbe15faf58 2024-12-06T15:03:43,131 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 0fa8eedb71b823cffb3dcfaededd9e04, 1588230740, 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 690f6a1423f23a78c40cb2559261b5c4, 69351ee18399c7dd21c5688d68629c27, b49cda5460cc4f0dddbd13c60efce4f7, e0189a0acac5efc8085840ccbd16c091, f23b08a30d88172abcf16915983abac9, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:43,136 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/info/e5c1e18fb0f5482aaa78a7cbe15faf58, entries=100, sequenceid=56, filesize=20.1 K 2024-12-06T15:03:43,139 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/ns/71de9e6ea4904cc391fd4a9161a89d72 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/ns/71de9e6ea4904cc391fd4a9161a89d72 2024-12-06T15:03:43,163 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/ns/71de9e6ea4904cc391fd4a9161a89d72, entries=2, sequenceid=56, filesize=5.0 K 2024-12-06T15:03:43,170 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-06T15:03:43,174 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/.tmp/table/ec47568bbbb9432c9160940c43bacf1c as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/table/ec47568bbbb9432c9160940c43bacf1c 2024-12-06T15:03:43,197 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/table/ec47568bbbb9432c9160940c43bacf1c, entries=20, sequenceid=56, filesize=6.5 K 2024-12-06T15:03:43,203 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 472ms, sequenceid=56, compaction requested=false 2024-12-06T15:03:43,227 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-06T15:03:43,229 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T15:03:43,229 INFO [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T15:03:43,229 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733497422730Running coprocessor pre-close hooks at 1733497422730Disabling compacts and flushes for region at 1733497422730Disabling writes for close at 1733497422731 (+1 ms)Obtaining lock to block concurrent updates at 1733497422733 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733497422733Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733497422741 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733497422743 (+2 ms)Flushing 1588230740/info: creating writer at 1733497422745 (+2 ms)Flushing 1588230740/info: appending metadata at 1733497422811 (+66 ms)Flushing 1588230740/info: closing flushed file at 1733497422811Flushing 1588230740/ns: creating writer at 1733497422940 (+129 ms)Flushing 1588230740/ns: appending metadata at 1733497422979 (+39 ms)Flushing 1588230740/ns: closing flushed file at 1733497422979Flushing 1588230740/table: creating writer at 1733497423037 (+58 ms)Flushing 1588230740/table: appending metadata at 1733497423079 (+42 ms)Flushing 1588230740/table: closing flushed file at 1733497423079Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@254f19b7: reopening flushed file at 1733497423121 (+42 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@402da42a: reopening flushed file at 1733497423136 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a115f24: reopening flushed file at 1733497423163 (+27 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 472ms, sequenceid=56, compaction requested=false at 1733497423204 (+41 ms)Writing region close event to WAL at 1733497423220 (+16 ms)Running coprocessor post-close hooks at 1733497423228 (+8 ms)Closed at 1733497423229 (+1 ms) 2024-12-06T15:03:43,229 DEBUG [RS_CLOSE_META-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T15:03:43,259 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T15:03:43,303 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-06T15:03:43,332 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 0fa8eedb71b823cffb3dcfaededd9e04, 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 690f6a1423f23a78c40cb2559261b5c4, 69351ee18399c7dd21c5688d68629c27, b49cda5460cc4f0dddbd13c60efce4f7, e0189a0acac5efc8085840ccbd16c091, f23b08a30d88172abcf16915983abac9, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:43,495 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04/.tmp/testFamily/82f4b60f637f41b0b899993c8b02dae6 2024-12-06T15:03:43,516 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04/.tmp/testFamily/82f4b60f637f41b0b899993c8b02dae6 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04/testFamily/82f4b60f637f41b0b899993c8b02dae6 2024-12-06T15:03:43,532 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 0fa8eedb71b823cffb3dcfaededd9e04, 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 690f6a1423f23a78c40cb2559261b5c4, 69351ee18399c7dd21c5688d68629c27, b49cda5460cc4f0dddbd13c60efce4f7, e0189a0acac5efc8085840ccbd16c091, f23b08a30d88172abcf16915983abac9, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:43,542 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04/testFamily/82f4b60f637f41b0b899993c8b02dae6, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:43,544 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0fa8eedb71b823cffb3dcfaededd9e04 in 513ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:43,560 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNext/0fa8eedb71b823cffb3dcfaededd9e04/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:43,563 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:43,564 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0fa8eedb71b823cffb3dcfaededd9e04: Waiting for close lock at 1733497423031Running coprocessor pre-close hooks at 1733497423031Disabling compacts and flushes for region at 1733497423031Disabling writes for close at 1733497423031Obtaining lock to block concurrent updates at 1733497423032 (+1 ms)Preparing flush snapshotting stores in 0fa8eedb71b823cffb3dcfaededd9e04 at 1733497423032Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497423032Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. at 1733497423033 (+1 ms)Flushing 0fa8eedb71b823cffb3dcfaededd9e04/testFamily: creating writer at 1733497423033Flushing 0fa8eedb71b823cffb3dcfaededd9e04/testFamily: appending metadata at 1733497423064 (+31 ms)Flushing 0fa8eedb71b823cffb3dcfaededd9e04/testFamily: closing flushed file at 1733497423064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d391f6: reopening flushed file at 1733497423514 (+450 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0fa8eedb71b823cffb3dcfaededd9e04 in 513ms, sequenceid=8, compaction requested=false at 1733497423544 (+30 ms)Writing region close event to WAL at 1733497423547 (+3 ms)Running coprocessor post-close hooks at 1733497423563 (+16 ms)Closed at 1733497423563 2024-12-06T15:03:43,564 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733497407503.0fa8eedb71b823cffb3dcfaededd9e04. 2024-12-06T15:03:43,564 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f23b08a30d88172abcf16915983abac9, disabling compactions & flushes 2024-12-06T15:03:43,564 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:43,564 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:43,564 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. after waiting 0 ms 2024-12-06T15:03:43,564 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:43,564 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f23b08a30d88172abcf16915983abac9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:43,591 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9/.tmp/testFamily/ae029892671a431584954822ee1326ac is 61, key is row-0/testFamily:testQualifier/1733497389291/Put/seqid=0 2024-12-06T15:03:43,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741860_1036 (size=5049) 2024-12-06T15:03:43,603 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9/.tmp/testFamily/ae029892671a431584954822ee1326ac 2024-12-06T15:03:43,622 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9/.tmp/testFamily/ae029892671a431584954822ee1326ac as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9/testFamily/ae029892671a431584954822ee1326ac 2024-12-06T15:03:43,639 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9/testFamily/ae029892671a431584954822ee1326ac, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:43,640 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f23b08a30d88172abcf16915983abac9 in 76ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:43,657 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnNextAsync/f23b08a30d88172abcf16915983abac9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:43,659 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:43,659 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f23b08a30d88172abcf16915983abac9: Waiting for close lock at 1733497423564Running coprocessor pre-close hooks at 1733497423564Disabling compacts and flushes for region at 1733497423564Disabling writes for close at 1733497423564Obtaining lock to block concurrent updates at 1733497423564Preparing flush snapshotting stores in f23b08a30d88172abcf16915983abac9 at 1733497423564Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497423565 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. at 1733497423568 (+3 ms)Flushing f23b08a30d88172abcf16915983abac9/testFamily: creating writer at 1733497423568Flushing f23b08a30d88172abcf16915983abac9/testFamily: appending metadata at 1733497423591 (+23 ms)Flushing f23b08a30d88172abcf16915983abac9/testFamily: closing flushed file at 1733497423591Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@517ddf9b: reopening flushed file at 1733497423620 (+29 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f23b08a30d88172abcf16915983abac9 in 76ms, sequenceid=8, compaction requested=false at 1733497423641 (+21 ms)Writing region close event to WAL at 1733497423644 (+3 ms)Running coprocessor post-close hooks at 1733497423658 (+14 ms)Closed at 1733497423658 2024-12-06T15:03:43,659 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733497386247.f23b08a30d88172abcf16915983abac9. 2024-12-06T15:03:43,659 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b49cda5460cc4f0dddbd13c60efce4f7, disabling compactions & flushes 2024-12-06T15:03:43,659 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:03:43,659 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:03:43,659 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. after waiting 0 ms 2024-12-06T15:03:43,660 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:03:43,660 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b49cda5460cc4f0dddbd13c60efce4f7 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:43,691 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7/.tmp/testFamily/0db9ca635acf438aa66d087c0976d9fc is 61, key is row-0/testFamily:testQualifier/1733497358746/Put/seqid=0 2024-12-06T15:03:43,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741861_1037 (size=5049) 2024-12-06T15:03:43,714 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7/.tmp/testFamily/0db9ca635acf438aa66d087c0976d9fc 2024-12-06T15:03:43,731 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7/.tmp/testFamily/0db9ca635acf438aa66d087c0976d9fc as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7/testFamily/0db9ca635acf438aa66d087c0976d9fc 2024-12-06T15:03:43,732 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-12-06T15:03:43,733 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1325): Online Regions={b49cda5460cc4f0dddbd13c60efce4f7=testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7., 690f6a1423f23a78c40cb2559261b5c4=hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4., 250bfe9d03810615ad16441d6f2803b2=testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., e0189a0acac5efc8085840ccbd16c091=hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., f8ce8bdc7344980bd3a5582a41147924=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., 29c7794be9ee9097a0bcfd6f62f3fe79=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., 69351ee18399c7dd21c5688d68629c27=testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27.} 2024-12-06T15:03:43,733 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 690f6a1423f23a78c40cb2559261b5c4, 69351ee18399c7dd21c5688d68629c27, b49cda5460cc4f0dddbd13c60efce4f7, e0189a0acac5efc8085840ccbd16c091, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:43,747 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7/testFamily/0db9ca635acf438aa66d087c0976d9fc, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:43,749 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b49cda5460cc4f0dddbd13c60efce4f7 in 89ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:43,762 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextExceptionAsync/b49cda5460cc4f0dddbd13c60efce4f7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:43,764 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:03:43,764 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b49cda5460cc4f0dddbd13c60efce4f7: Waiting for close lock at 1733497423659Running coprocessor pre-close hooks at 1733497423659Disabling compacts and flushes for region at 1733497423659Disabling writes for close at 1733497423660 (+1 ms)Obtaining lock to block concurrent updates at 1733497423660Preparing flush snapshotting stores in b49cda5460cc4f0dddbd13c60efce4f7 at 1733497423660Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497423660Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. at 1733497423664 (+4 ms)Flushing b49cda5460cc4f0dddbd13c60efce4f7/testFamily: creating writer at 1733497423664Flushing b49cda5460cc4f0dddbd13c60efce4f7/testFamily: appending metadata at 1733497423690 (+26 ms)Flushing b49cda5460cc4f0dddbd13c60efce4f7/testFamily: closing flushed file at 1733497423690Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@142c73ca: reopening flushed file at 1733497423728 (+38 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b49cda5460cc4f0dddbd13c60efce4f7 in 89ms, sequenceid=8, compaction requested=false at 1733497423749 (+21 ms)Writing region close event to WAL at 1733497423753 (+4 ms)Running coprocessor post-close hooks at 1733497423763 (+10 ms)Closed at 1733497423764 (+1 ms) 2024-12-06T15:03:43,764 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733497357688.b49cda5460cc4f0dddbd13c60efce4f7. 2024-12-06T15:03:43,764 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 690f6a1423f23a78c40cb2559261b5c4, disabling compactions & flushes 2024-12-06T15:03:43,764 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:03:43,764 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:03:43,764 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. after waiting 0 ms 2024-12-06T15:03:43,764 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:03:43,764 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 690f6a1423f23a78c40cb2559261b5c4 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:43,789 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4/.tmp/testFamily/bb62685e2bd4465e99215e565e71ac1d is 61, key is row-0/testFamily:testQualifier/1733497370994/Put/seqid=0 2024-12-06T15:03:43,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741862_1038 (size=5049) 2024-12-06T15:03:43,801 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4/.tmp/testFamily/bb62685e2bd4465e99215e565e71ac1d 2024-12-06T15:03:43,812 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4/.tmp/testFamily/bb62685e2bd4465e99215e565e71ac1d as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4/testFamily/bb62685e2bd4465e99215e565e71ac1d 2024-12-06T15:03:43,825 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4/testFamily/bb62685e2bd4465e99215e565e71ac1d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:43,827 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 690f6a1423f23a78c40cb2559261b5c4 in 62ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:43,840 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnNextAsync/690f6a1423f23a78c40cb2559261b5c4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:43,841 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:03:43,841 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 690f6a1423f23a78c40cb2559261b5c4: Waiting for close lock at 1733497423764Running coprocessor pre-close hooks at 1733497423764Disabling compacts and flushes for region at 1733497423764Disabling writes for close at 1733497423764Obtaining lock to block concurrent updates at 1733497423764Preparing flush snapshotting stores in 690f6a1423f23a78c40cb2559261b5c4 at 1733497423764Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497423765 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. at 1733497423765Flushing 690f6a1423f23a78c40cb2559261b5c4/testFamily: creating writer at 1733497423766 (+1 ms)Flushing 690f6a1423f23a78c40cb2559261b5c4/testFamily: appending metadata at 1733497423788 (+22 ms)Flushing 690f6a1423f23a78c40cb2559261b5c4/testFamily: closing flushed file at 1733497423788Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2056a513: reopening flushed file at 1733497423810 (+22 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 690f6a1423f23a78c40cb2559261b5c4 in 62ms, sequenceid=8, compaction requested=false at 1733497423827 (+17 ms)Writing region close event to WAL at 1733497423827Running coprocessor post-close hooks at 1733497423841 (+14 ms)Closed at 1733497423841 2024-12-06T15:03:43,841 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733497367946.690f6a1423f23a78c40cb2559261b5c4. 2024-12-06T15:03:43,841 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 250bfe9d03810615ad16441d6f2803b2, disabling compactions & flushes 2024-12-06T15:03:43,841 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:43,842 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:43,842 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. after waiting 0 ms 2024-12-06T15:03:43,842 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:43,842 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 250bfe9d03810615ad16441d6f2803b2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:43,870 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2/.tmp/testFamily/76515dde47fa4ea2b0ccedb3f9f5e936 is 61, key is row-0/testFamily:testQualifier/1733497383200/Put/seqid=0 2024-12-06T15:03:43,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741863_1039 (size=5049) 2024-12-06T15:03:43,933 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 69351ee18399c7dd21c5688d68629c27, e0189a0acac5efc8085840ccbd16c091, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:44,013 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-06T15:03:44,013 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-06T15:03:44,014 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T15:03:44,014 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-06T15:03:44,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-06T15:03:44,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T15:03:44,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-06T15:03:44,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-06T15:03:44,015 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-06T15:03:44,133 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 250bfe9d03810615ad16441d6f2803b2, 29c7794be9ee9097a0bcfd6f62f3fe79, 69351ee18399c7dd21c5688d68629c27, e0189a0acac5efc8085840ccbd16c091, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:44,277 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2/.tmp/testFamily/76515dde47fa4ea2b0ccedb3f9f5e936 2024-12-06T15:03:44,287 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2/.tmp/testFamily/76515dde47fa4ea2b0ccedb3f9f5e936 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2/testFamily/76515dde47fa4ea2b0ccedb3f9f5e936 2024-12-06T15:03:44,297 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2/testFamily/76515dde47fa4ea2b0ccedb3f9f5e936, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:44,298 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 250bfe9d03810615ad16441d6f2803b2 in 456ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:44,304 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScanner/250bfe9d03810615ad16441d6f2803b2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:44,305 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:44,305 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 250bfe9d03810615ad16441d6f2803b2: Waiting for close lock at 1733497423841Running coprocessor pre-close hooks at 1733497423841Disabling compacts and flushes for region at 1733497423841Disabling writes for close at 1733497423842 (+1 ms)Obtaining lock to block concurrent updates at 1733497423842Preparing flush snapshotting stores in 250bfe9d03810615ad16441d6f2803b2 at 1733497423842Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497423842Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. at 1733497423843 (+1 ms)Flushing 250bfe9d03810615ad16441d6f2803b2/testFamily: creating writer at 1733497423843Flushing 250bfe9d03810615ad16441d6f2803b2/testFamily: appending metadata at 1733497423869 (+26 ms)Flushing 250bfe9d03810615ad16441d6f2803b2/testFamily: closing flushed file at 1733497423870 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5226907b: reopening flushed file at 1733497424286 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 250bfe9d03810615ad16441d6f2803b2 in 456ms, sequenceid=8, compaction requested=false at 1733497424298 (+12 ms)Writing region close event to WAL at 1733497424299 (+1 ms)Running coprocessor post-close hooks at 1733497424305 (+6 ms)Closed at 1733497424305 2024-12-06T15:03:44,305 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733497380149.250bfe9d03810615ad16441d6f2803b2. 2024-12-06T15:03:44,305 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e0189a0acac5efc8085840ccbd16c091, disabling compactions & flushes 2024-12-06T15:03:44,306 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:03:44,306 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:03:44,306 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. after waiting 0 ms 2024-12-06T15:03:44,306 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:03:44,306 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e0189a0acac5efc8085840ccbd16c091 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:44,329 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091/.tmp/testFamily/459a79e3e30a4b9f8dc8487cc92f9c88 is 61, key is row-0/testFamily:testQualifier/1733497361893/Put/seqid=0 2024-12-06T15:03:44,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741864_1040 (size=5049) 2024-12-06T15:03:44,334 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 29c7794be9ee9097a0bcfd6f62f3fe79, 69351ee18399c7dd21c5688d68629c27, e0189a0acac5efc8085840ccbd16c091, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:44,340 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091/.tmp/testFamily/459a79e3e30a4b9f8dc8487cc92f9c88 2024-12-06T15:03:44,349 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091/.tmp/testFamily/459a79e3e30a4b9f8dc8487cc92f9c88 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091/testFamily/459a79e3e30a4b9f8dc8487cc92f9c88 2024-12-06T15:03:44,358 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091/testFamily/459a79e3e30a4b9f8dc8487cc92f9c88, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:44,360 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e0189a0acac5efc8085840ccbd16c091 in 54ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:44,365 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScanner/e0189a0acac5efc8085840ccbd16c091/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:44,366 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:03:44,366 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e0189a0acac5efc8085840ccbd16c091: Waiting for close lock at 1733497424305Running coprocessor pre-close hooks at 1733497424305Disabling compacts and flushes for region at 1733497424305Disabling writes for close at 1733497424306 (+1 ms)Obtaining lock to block concurrent updates at 1733497424306Preparing flush snapshotting stores in e0189a0acac5efc8085840ccbd16c091 at 1733497424306Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497424306Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. at 1733497424307 (+1 ms)Flushing e0189a0acac5efc8085840ccbd16c091/testFamily: creating writer at 1733497424307Flushing e0189a0acac5efc8085840ccbd16c091/testFamily: appending metadata at 1733497424328 (+21 ms)Flushing e0189a0acac5efc8085840ccbd16c091/testFamily: closing flushed file at 1733497424329 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c814c90: reopening flushed file at 1733497424348 (+19 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e0189a0acac5efc8085840ccbd16c091 in 54ms, sequenceid=8, compaction requested=false at 1733497424360 (+12 ms)Writing region close event to WAL at 1733497424361 (+1 ms)Running coprocessor post-close hooks at 1733497424366 (+5 ms)Closed at 1733497424366 2024-12-06T15:03:44,366 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733497358835.e0189a0acac5efc8085840ccbd16c091. 2024-12-06T15:03:44,366 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f8ce8bdc7344980bd3a5582a41147924, disabling compactions & flushes 2024-12-06T15:03:44,366 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:44,367 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:44,367 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. after waiting 0 ms 2024-12-06T15:03:44,367 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:44,367 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f8ce8bdc7344980bd3a5582a41147924 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:44,384 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924/.tmp/testFamily/37629b0760084f6b8eeddc9576b6132a is 61, key is row-0/testFamily:testQualifier/1733497401461/Put/seqid=0 2024-12-06T15:03:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741865_1041 (size=5049) 2024-12-06T15:03:44,534 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 29c7794be9ee9097a0bcfd6f62f3fe79, 69351ee18399c7dd21c5688d68629c27, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:44,734 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-06T15:03:44,734 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1325): Online Regions={f8ce8bdc7344980bd3a5582a41147924=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., 29c7794be9ee9097a0bcfd6f62f3fe79=testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., 69351ee18399c7dd21c5688d68629c27=testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27.} 2024-12-06T15:03:44,734 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 29c7794be9ee9097a0bcfd6f62f3fe79, 69351ee18399c7dd21c5688d68629c27, f8ce8bdc7344980bd3a5582a41147924 2024-12-06T15:03:44,791 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924/.tmp/testFamily/37629b0760084f6b8eeddc9576b6132a 2024-12-06T15:03:44,801 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924/.tmp/testFamily/37629b0760084f6b8eeddc9576b6132a as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924/testFamily/37629b0760084f6b8eeddc9576b6132a 2024-12-06T15:03:44,809 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924/testFamily/37629b0760084f6b8eeddc9576b6132a, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:44,810 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f8ce8bdc7344980bd3a5582a41147924 in 443ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:44,815 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/f8ce8bdc7344980bd3a5582a41147924/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:44,816 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:44,816 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f8ce8bdc7344980bd3a5582a41147924: Waiting for close lock at 1733497424366Running coprocessor pre-close hooks at 1733497424366Disabling compacts and flushes for region at 1733497424366Disabling writes for close at 1733497424367 (+1 ms)Obtaining lock to block concurrent updates at 1733497424367Preparing flush snapshotting stores in f8ce8bdc7344980bd3a5582a41147924 at 1733497424367Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497424367Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. at 1733497424368 (+1 ms)Flushing f8ce8bdc7344980bd3a5582a41147924/testFamily: creating writer at 1733497424368Flushing f8ce8bdc7344980bd3a5582a41147924/testFamily: appending metadata at 1733497424383 (+15 ms)Flushing f8ce8bdc7344980bd3a5582a41147924/testFamily: closing flushed file at 1733497424383Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542959c5: reopening flushed file at 1733497424799 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f8ce8bdc7344980bd3a5582a41147924 in 443ms, sequenceid=8, compaction requested=false at 1733497424810 (+11 ms)Writing region close event to WAL at 1733497424811 (+1 ms)Running coprocessor post-close hooks at 1733497424816 (+5 ms)Closed at 1733497424816 2024-12-06T15:03:44,816 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733497398419.f8ce8bdc7344980bd3a5582a41147924. 2024-12-06T15:03:44,816 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 29c7794be9ee9097a0bcfd6f62f3fe79, disabling compactions & flushes 2024-12-06T15:03:44,817 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:44,817 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:44,817 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. after waiting 0 ms 2024-12-06T15:03:44,817 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:44,817 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 29c7794be9ee9097a0bcfd6f62f3fe79 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:44,833 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79/.tmp/testFamily/6c5ebfa5144e4bea85e2a8cf20fe7cf3 is 61, key is row-0/testFamily:testQualifier/1733497419640/Put/seqid=0 2024-12-06T15:03:44,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741866_1042 (size=5049) 2024-12-06T15:03:44,935 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 29c7794be9ee9097a0bcfd6f62f3fe79, 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:03:45,135 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 29c7794be9ee9097a0bcfd6f62f3fe79, 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:03:45,239 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79/.tmp/testFamily/6c5ebfa5144e4bea85e2a8cf20fe7cf3 2024-12-06T15:03:45,249 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79/.tmp/testFamily/6c5ebfa5144e4bea85e2a8cf20fe7cf3 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79/testFamily/6c5ebfa5144e4bea85e2a8cf20fe7cf3 2024-12-06T15:03:45,258 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79/testFamily/6c5ebfa5144e4bea85e2a8cf20fe7cf3, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:45,259 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 29c7794be9ee9097a0bcfd6f62f3fe79 in 442ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:45,264 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testNormalScanTimeoutOnOpenScannerAsync/29c7794be9ee9097a0bcfd6f62f3fe79/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:45,265 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:45,265 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 29c7794be9ee9097a0bcfd6f62f3fe79: Waiting for close lock at 1733497424816Running coprocessor pre-close hooks at 1733497424816Disabling compacts and flushes for region at 1733497424816Disabling writes for close at 1733497424817 (+1 ms)Obtaining lock to block concurrent updates at 1733497424817Preparing flush snapshotting stores in 29c7794be9ee9097a0bcfd6f62f3fe79 at 1733497424817Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497424817Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. at 1733497424818 (+1 ms)Flushing 29c7794be9ee9097a0bcfd6f62f3fe79/testFamily: creating writer at 1733497424818Flushing 29c7794be9ee9097a0bcfd6f62f3fe79/testFamily: appending metadata at 1733497424833 (+15 ms)Flushing 29c7794be9ee9097a0bcfd6f62f3fe79/testFamily: closing flushed file at 1733497424833Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7571d470: reopening flushed file at 1733497425247 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 29c7794be9ee9097a0bcfd6f62f3fe79 in 442ms, sequenceid=8, compaction requested=false at 1733497425259 (+12 ms)Writing region close event to WAL at 1733497425260 (+1 ms)Running coprocessor post-close hooks at 1733497425265 (+5 ms)Closed at 1733497425265 2024-12-06T15:03:45,265 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733497416592.29c7794be9ee9097a0bcfd6f62f3fe79. 2024-12-06T15:03:45,265 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69351ee18399c7dd21c5688d68629c27, disabling compactions & flushes 2024-12-06T15:03:45,266 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:03:45,266 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:03:45,266 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. after waiting 0 ms 2024-12-06T15:03:45,266 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:03:45,266 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 69351ee18399c7dd21c5688d68629c27 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-06T15:03:45,283 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27/.tmp/testFamily/de6165e41a894f0e8fdafcec33b4d6da is 61, key is row-0/testFamily:testQualifier/1733497380093/Put/seqid=0 2024-12-06T15:03:45,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741867_1043 (size=5049) 2024-12-06T15:03:45,335 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:03:45,535 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1351): Waiting on 69351ee18399c7dd21c5688d68629c27 2024-12-06T15:03:45,689 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27/.tmp/testFamily/de6165e41a894f0e8fdafcec33b4d6da 2024-12-06T15:03:45,697 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27/.tmp/testFamily/de6165e41a894f0e8fdafcec33b4d6da as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27/testFamily/de6165e41a894f0e8fdafcec33b4d6da 2024-12-06T15:03:45,705 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27/testFamily/de6165e41a894f0e8fdafcec33b4d6da, entries=4, sequenceid=8, filesize=4.9 K 2024-12-06T15:03:45,706 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 69351ee18399c7dd21c5688d68629c27 in 440ms, sequenceid=8, compaction requested=false 2024-12-06T15:03:45,711 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/data/default/testRetryOutOfOrderScannerNextException/69351ee18399c7dd21c5688d68629c27/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-06T15:03:45,712 INFO [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:03:45,712 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69351ee18399c7dd21c5688d68629c27: Waiting for close lock at 1733497425265Running coprocessor pre-close hooks at 1733497425265Disabling compacts and flushes for region at 1733497425265Disabling writes for close at 1733497425266 (+1 ms)Obtaining lock to block concurrent updates at 1733497425266Preparing flush snapshotting stores in 69351ee18399c7dd21c5688d68629c27 at 1733497425266Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733497425266Flushing stores of testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. at 1733497425267 (+1 ms)Flushing 69351ee18399c7dd21c5688d68629c27/testFamily: creating writer at 1733497425267Flushing 69351ee18399c7dd21c5688d68629c27/testFamily: appending metadata at 1733497425283 (+16 ms)Flushing 69351ee18399c7dd21c5688d68629c27/testFamily: closing flushed file at 1733497425283Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65f4ce2b: reopening flushed file at 1733497425695 (+412 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 69351ee18399c7dd21c5688d68629c27 in 440ms, sequenceid=8, compaction requested=false at 1733497425706 (+11 ms)Writing region close event to WAL at 1733497425707 (+1 ms)Running coprocessor post-close hooks at 1733497425712 (+5 ms)Closed at 1733497425712 2024-12-06T15:03:45,712 DEBUG [RS_CLOSE_REGION-regionserver/bb7ea3e46d9c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733497377044.69351ee18399c7dd21c5688d68629c27. 2024-12-06T15:03:45,736 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(976): stopping server bb7ea3e46d9c,45829,1733497324439; all regions closed. 2024-12-06T15:03:45,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741834_1010 (size=19479) 2024-12-06T15:03:45,751 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/oldWALs 2024-12-06T15:03:45,751 INFO [RS:0;bb7ea3e46d9c:45829 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C45829%2C1733497324439.meta:.meta(num 1733497357230) 2024-12-06T15:03:45,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741833_1009 (size=20031) 2024-12-06T15:03:45,756 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/oldWALs 2024-12-06T15:03:45,757 INFO [RS:0;bb7ea3e46d9c:45829 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bb7ea3e46d9c%2C45829%2C1733497324439:(num 1733497356714) 2024-12-06T15:03:45,757 DEBUG [RS:0;bb7ea3e46d9c:45829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T15:03:45,757 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T15:03:45,757 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T15:03:45,757 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.ChoreService(370): Chore service for: regionserver/bb7ea3e46d9c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T15:03:45,758 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T15:03:45,758 INFO [regionserver/bb7ea3e46d9c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T15:03:45,759 INFO [RS:0;bb7ea3e46d9c:45829 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45829 2024-12-06T15:03:45,782 INFO [regionserver/bb7ea3e46d9c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T15:03:45,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T15:03:45,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bb7ea3e46d9c,45829,1733497324439 2024-12-06T15:03:45,782 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T15:03:45,783 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bb7ea3e46d9c,45829,1733497324439] 2024-12-06T15:03:45,803 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bb7ea3e46d9c,45829,1733497324439 already deleted, retry=false 2024-12-06T15:03:45,803 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bb7ea3e46d9c,45829,1733497324439 expired; onlineServers=0 2024-12-06T15:03:45,804 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bb7ea3e46d9c,43691,1733497323635' ***** 2024-12-06T15:03:45,804 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T15:03:45,804 INFO [M:0;bb7ea3e46d9c:43691 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T15:03:45,804 INFO [M:0;bb7ea3e46d9c:43691 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T15:03:45,804 DEBUG [M:0;bb7ea3e46d9c:43691 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T15:03:45,804 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T15:03:45,804 DEBUG [M:0;bb7ea3e46d9c:43691 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T15:03:45,804 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733497326534 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.large.0-1733497326534,5,FailOnTimeoutGroup] 2024-12-06T15:03:45,804 DEBUG [master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733497326536 {}] cleaner.HFileCleaner(306): Exit Thread[master/bb7ea3e46d9c:0:becomeActiveMaster-HFileCleaner.small.0-1733497326536,5,FailOnTimeoutGroup] 2024-12-06T15:03:45,804 INFO [M:0;bb7ea3e46d9c:43691 {}] hbase.ChoreService(370): Chore service for: master/bb7ea3e46d9c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T15:03:45,805 INFO [M:0;bb7ea3e46d9c:43691 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T15:03:45,805 DEBUG [M:0;bb7ea3e46d9c:43691 {}] master.HMaster(1795): Stopping service threads 2024-12-06T15:03:45,805 INFO [M:0;bb7ea3e46d9c:43691 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T15:03:45,805 INFO [M:0;bb7ea3e46d9c:43691 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T15:03:45,806 INFO [M:0;bb7ea3e46d9c:43691 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T15:03:45,806 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T15:03:45,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T15:03:45,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T15:03:45,818 DEBUG [M:0;bb7ea3e46d9c:43691 {}] zookeeper.ZKUtil(347): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T15:03:45,818 WARN [M:0;bb7ea3e46d9c:43691 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T15:03:45,819 INFO [M:0;bb7ea3e46d9c:43691 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/.lastflushedseqids 2024-12-06T15:03:45,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741868_1044 (size=649) 2024-12-06T15:03:45,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T15:03:45,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45829-0x101ab29f8bc0001, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T15:03:45,894 INFO [RS:0;bb7ea3e46d9c:45829 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T15:03:45,894 INFO [RS:0;bb7ea3e46d9c:45829 {}] regionserver.HRegionServer(1031): Exiting; stopping=bb7ea3e46d9c,45829,1733497324439; zookeeper connection closed. 2024-12-06T15:03:45,894 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1045b1c2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1045b1c2 2024-12-06T15:03:45,895 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-06T15:03:46,230 INFO [M:0;bb7ea3e46d9c:43691 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T15:03:46,230 INFO [M:0;bb7ea3e46d9c:43691 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T15:03:46,230 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T15:03:46,230 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:03:46,230 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:03:46,230 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T15:03:46,230 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:03:46,231 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.99 KB heapSize=190.90 KB 2024-12-06T15:03:46,250 DEBUG [M:0;bb7ea3e46d9c:43691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe952984805845d687ae345b8c1f5ab5 is 82, key is hbase:meta,,1/info:regioninfo/1733497357310/Put/seqid=0 2024-12-06T15:03:46,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741869_1045 (size=5672) 2024-12-06T15:03:46,657 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe952984805845d687ae345b8c1f5ab5 2024-12-06T15:03:46,686 DEBUG [M:0;bb7ea3e46d9c:43691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95158747bb674576b2efb15aded0184d is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733497358542/Put/seqid=0 2024-12-06T15:03:46,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741870_1046 (size=14069) 2024-12-06T15:03:47,093 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95158747bb674576b2efb15aded0184d 2024-12-06T15:03:47,106 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95158747bb674576b2efb15aded0184d 2024-12-06T15:03:47,123 DEBUG [M:0;bb7ea3e46d9c:43691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/32779535c5a447089ff90099626c8f10 is 69, key is bb7ea3e46d9c,45829,1733497324439/rs:state/1733497326579/Put/seqid=0 2024-12-06T15:03:47,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741871_1047 (size=5156) 2024-12-06T15:03:47,528 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/32779535c5a447089ff90099626c8f10 2024-12-06T15:03:47,540 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe952984805845d687ae345b8c1f5ab5 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe952984805845d687ae345b8c1f5ab5 2024-12-06T15:03:47,551 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe952984805845d687ae345b8c1f5ab5, entries=8, sequenceid=317, filesize=5.5 K 2024-12-06T15:03:47,553 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95158747bb674576b2efb15aded0184d as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95158747bb674576b2efb15aded0184d 2024-12-06T15:03:47,560 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95158747bb674576b2efb15aded0184d 2024-12-06T15:03:47,561 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95158747bb674576b2efb15aded0184d, entries=33, sequenceid=317, filesize=13.7 K 2024-12-06T15:03:47,562 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/32779535c5a447089ff90099626c8f10 as hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/32779535c5a447089ff90099626c8f10 2024-12-06T15:03:47,570 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35547/user/jenkins/test-data/3fc12a51-d40c-65d0-fe5e-9c5dfea3dbaf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/32779535c5a447089ff90099626c8f10, entries=1, sequenceid=317, filesize=5.0 K 2024-12-06T15:03:47,571 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.99 KB/166903, heapSize ~190.60 KB/195176, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1341ms, sequenceid=317, compaction requested=false 2024-12-06T15:03:47,576 INFO [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T15:03:47,576 DEBUG [M:0;bb7ea3e46d9c:43691 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733497426230Disabling compacts and flushes for region at 1733497426230Disabling writes for close at 1733497426230Obtaining lock to block concurrent updates at 1733497426231 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733497426231Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166903, getHeapSize=195416, getOffHeapSize=0, getCellsCount=364 at 1733497426231Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733497426232 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733497426232Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733497426250 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733497426250Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733497426666 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733497426685 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733497426685Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733497427106 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733497427122 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733497427122Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5df2f253: reopening flushed file at 1733497427537 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f67bcc9: reopening flushed file at 1733497427551 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f274e70: reopening flushed file at 1733497427561 (+10 ms)Finished flush of dataSize ~162.99 KB/166903, heapSize ~190.60 KB/195176, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1341ms, sequenceid=317, compaction requested=false at 1733497427571 (+10 ms)Writing region close event to WAL at 1733497427576 (+5 ms)Closed at 1733497427576 2024-12-06T15:03:47,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42777 is added to blk_1073741830_1006 (size=188115) 2024-12-06T15:03:47,582 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T15:03:47,582 INFO [M:0;bb7ea3e46d9c:43691 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T15:03:47,582 INFO [M:0;bb7ea3e46d9c:43691 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43691 2024-12-06T15:03:47,583 INFO [M:0;bb7ea3e46d9c:43691 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T15:03:47,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T15:03:47,714 INFO [M:0;bb7ea3e46d9c:43691 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T15:03:47,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43691-0x101ab29f8bc0000, quorum=127.0.0.1:50293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T15:03:47,719 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T15:03:47,723 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:03:47,723 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:03:47,723 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:03:47,723 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/hadoop.log.dir/,STOPPED} 2024-12-06T15:03:47,726 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T15:03:47,726 WARN [BP-1257781960-172.17.0.2-1733497319399 heartbeating to localhost/127.0.0.1:35547 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T15:03:47,727 WARN [BP-1257781960-172.17.0.2-1733497319399 heartbeating to localhost/127.0.0.1:35547 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1257781960-172.17.0.2-1733497319399 (Datanode Uuid fd935b72-4700-4b0b-880c-d48f79917280) service to localhost/127.0.0.1:35547 2024-12-06T15:03:47,727 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T15:03:47,728 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207/data/data1/current/BP-1257781960-172.17.0.2-1733497319399 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:03:47,729 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/cluster_ba37c31a-b7ec-4458-0c69-e7b408932207/data/data2/current/BP-1257781960-172.17.0.2-1733497319399 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T15:03:47,729 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T15:03:47,740 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T15:03:47,741 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T15:03:47,741 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T15:03:47,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T15:03:47,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3605fbf3-d9b0-9546-1d94-dafbca99e22b/hadoop.log.dir/,STOPPED} 2024-12-06T15:03:47,750 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T15:03:47,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down