2024-11-27 03:39:39,436 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-27 03:39:39,457 main DEBUG Took 0.018847 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 03:39:39,458 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 03:39:39,458 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 03:39:39,459 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 03:39:39,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,469 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 03:39:39,485 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,486 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,487 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,488 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,490 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,492 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,493 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,494 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,495 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,496 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,496 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,497 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,498 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,499 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,499 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,499 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 03:39:39,500 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,500 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 03:39:39,502 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 03:39:39,503 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 03:39:39,506 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 03:39:39,507 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 03:39:39,508 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 03:39:39,509 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 03:39:39,519 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 03:39:39,522 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 03:39:39,524 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 03:39:39,525 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 03:39:39,525 main DEBUG createAppenders(={Console}) 2024-11-27 03:39:39,526 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-27 03:39:39,527 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-27 03:39:39,527 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-27 03:39:39,528 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 03:39:39,529 main DEBUG OutputStream closed 2024-11-27 03:39:39,529 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 03:39:39,529 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 03:39:39,530 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-27 03:39:39,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 03:39:39,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 03:39:39,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 03:39:39,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 03:39:39,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 03:39:39,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 03:39:39,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 03:39:39,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 03:39:39,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 03:39:39,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 03:39:39,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 03:39:39,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 03:39:39,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 03:39:39,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 03:39:39,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 03:39:39,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 03:39:39,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 03:39:39,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 03:39:39,638 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 03:39:39,639 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-27 03:39:39,639 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 03:39:39,640 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-27T03:39:39,654 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-27 03:39:39,657 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 03:39:39,657 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T03:39:39,942 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d 2024-11-27T03:39:39,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T03:39:39,990 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10, deleteOnExit=true 2024-11-27T03:39:39,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T03:39:39,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/test.cache.data in system properties and HBase conf 2024-11-27T03:39:39,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T03:39:39,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/hadoop.log.dir in system properties and HBase conf 2024-11-27T03:39:39,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T03:39:39,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T03:39:39,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T03:39:40,114 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T03:39:40,219 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T03:39:40,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T03:39:40,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T03:39:40,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T03:39:40,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T03:39:40,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T03:39:40,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T03:39:40,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T03:39:40,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T03:39:40,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T03:39:40,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/nfs.dump.dir in system properties and HBase conf 2024-11-27T03:39:40,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/java.io.tmpdir in system properties and HBase conf 2024-11-27T03:39:40,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T03:39:40,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T03:39:40,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T03:39:41,404 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T03:39:41,499 INFO [Time-limited test {}] log.Log(170): Logging initialized @2869ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T03:39:41,580 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T03:39:41,653 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T03:39:41,680 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T03:39:41,680 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T03:39:41,682 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T03:39:41,696 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T03:39:41,699 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/hadoop.log.dir/,AVAILABLE} 2024-11-27T03:39:41,700 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T03:39:41,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4df87c21{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/java.io.tmpdir/jetty-localhost-46057-hadoop-hdfs-3_4_1-tests_jar-_-any-1019750862859027735/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T03:39:41,933 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:46057} 2024-11-27T03:39:41,933 INFO [Time-limited test {}] server.Server(415): Started @3303ms 2024-11-27T03:39:42,570 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T03:39:42,580 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T03:39:42,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T03:39:42,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T03:39:42,582 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T03:39:42,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7aa82d16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/hadoop.log.dir/,AVAILABLE} 2024-11-27T03:39:42,584 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23c598c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T03:39:42,710 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f9a1a82{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/java.io.tmpdir/jetty-localhost-33887-hadoop-hdfs-3_4_1-tests_jar-_-any-8390814835834640610/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T03:39:42,711 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@210254cd{HTTP/1.1, (http/1.1)}{localhost:33887} 2024-11-27T03:39:42,711 INFO [Time-limited test {}] server.Server(415): Started @4082ms 2024-11-27T03:39:42,779 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T03:39:43,950 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10/data/data1/current/BP-1930725240-172.17.0.3-1732678780861/current, will proceed with Du for space computation calculation, 2024-11-27T03:39:43,950 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10/data/data2/current/BP-1930725240-172.17.0.3-1732678780861/current, will proceed with Du for space computation calculation, 2024-11-27T03:39:43,989 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T03:39:44,042 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d 2024-11-27T03:39:44,048 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x60c78b13a7237a78 with lease ID 0x8996fe43ecd58dbe: Processing first storage report for DS-870dfb75-f227-4af3-bac1-7ad017699541 from datanode DatanodeRegistration(127.0.0.1:34855, datanodeUuid=9c1f62c8-f8b1-4269-93ae-d9bab8226688, infoPort=40389, infoSecurePort=0, ipcPort=45549, storageInfo=lv=-57;cid=testClusterID;nsid=923434038;c=1732678780861) 2024-11-27T03:39:44,049 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60c78b13a7237a78 with lease ID 0x8996fe43ecd58dbe: from storage DS-870dfb75-f227-4af3-bac1-7ad017699541 node DatanodeRegistration(127.0.0.1:34855, datanodeUuid=9c1f62c8-f8b1-4269-93ae-d9bab8226688, infoPort=40389, infoSecurePort=0, ipcPort=45549, storageInfo=lv=-57;cid=testClusterID;nsid=923434038;c=1732678780861), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T03:39:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x60c78b13a7237a78 with lease ID 0x8996fe43ecd58dbe: Processing first storage report for DS-e9593d8b-9fff-42d9-ae18-9a795db95953 from datanode DatanodeRegistration(127.0.0.1:34855, datanodeUuid=9c1f62c8-f8b1-4269-93ae-d9bab8226688, infoPort=40389, infoSecurePort=0, ipcPort=45549, storageInfo=lv=-57;cid=testClusterID;nsid=923434038;c=1732678780861) 2024-11-27T03:39:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x60c78b13a7237a78 with lease ID 0x8996fe43ecd58dbe: from storage DS-e9593d8b-9fff-42d9-ae18-9a795db95953 node DatanodeRegistration(127.0.0.1:34855, datanodeUuid=9c1f62c8-f8b1-4269-93ae-d9bab8226688, infoPort=40389, infoSecurePort=0, ipcPort=45549, storageInfo=lv=-57;cid=testClusterID;nsid=923434038;c=1732678780861), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T03:39:44,150 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10/zookeeper_0, clientPort=63169, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T03:39:44,165 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63169 2024-11-27T03:39:44,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T03:39:44,184 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T03:39:44,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741825_1001 (size=7) 2024-11-27T03:39:44,843 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e with version=8 2024-11-27T03:39:44,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/hbase-staging 2024-11-27T03:39:44,936 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T03:39:45,201 INFO [Time-limited test {}] client.ConnectionUtils(128): master/66d3634da959:0 server-side Connection retries=9 2024-11-27T03:39:45,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T03:39:45,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T03:39:45,217 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T03:39:45,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T03:39:45,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T03:39:45,372 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T03:39:45,433 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T03:39:45,443 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T03:39:45,447 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T03:39:45,472 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 104906 (auto-detected) 2024-11-27T03:39:45,473 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-27T03:39:45,493 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35691 2024-11-27T03:39:45,514 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35691 connecting to ZooKeeper ensemble=127.0.0.1:63169 2024-11-27T03:39:45,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356910x0, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T03:39:45,637 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35691-0x1017a6010240000 connected 2024-11-27T03:39:45,734 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T03:39:45,737 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T03:39:45,749 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T03:39:45,754 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e, hbase.cluster.distributed=false 2024-11-27T03:39:45,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T03:39:45,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35691 2024-11-27T03:39:45,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35691 2024-11-27T03:39:45,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35691 2024-11-27T03:39:45,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35691 2024-11-27T03:39:45,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35691 2024-11-27T03:39:45,920 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/66d3634da959:0 server-side Connection retries=9 2024-11-27T03:39:45,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T03:39:45,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T03:39:45,922 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T03:39:45,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T03:39:45,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T03:39:45,926 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T03:39:45,929 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T03:39:45,931 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43983 2024-11-27T03:39:45,933 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43983 connecting to ZooKeeper ensemble=127.0.0.1:63169 2024-11-27T03:39:45,935 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T03:39:45,939 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T03:39:45,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:439830x0, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T03:39:45,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43983-0x1017a6010240001 connected 2024-11-27T03:39:45,958 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T03:39:45,964 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T03:39:45,976 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T03:39:45,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T03:39:45,985 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T03:39:45,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43983 2024-11-27T03:39:45,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43983 2024-11-27T03:39:45,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43983 2024-11-27T03:39:45,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43983 2024-11-27T03:39:45,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43983 2024-11-27T03:39:46,009 DEBUG [M:0;66d3634da959:35691 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;66d3634da959:35691 2024-11-27T03:39:46,010 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/66d3634da959,35691,1732678785028 2024-11-27T03:39:46,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T03:39:46,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T03:39:46,030 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/66d3634da959,35691,1732678785028 2024-11-27T03:39:46,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T03:39:46,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:46,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:46,069 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T03:39:46,070 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/66d3634da959,35691,1732678785028 from backup master directory 2024-11-27T03:39:46,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/66d3634da959,35691,1732678785028 2024-11-27T03:39:46,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T03:39:46,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T03:39:46,080 WARN [master/66d3634da959:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T03:39:46,080 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=66d3634da959,35691,1732678785028 2024-11-27T03:39:46,082 INFO [master/66d3634da959:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T03:39:46,084 INFO [master/66d3634da959:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T03:39:46,152 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/hbase.id] with ID: 70b3580d-2862-4778-8ac5-3cbc57e5b9a3 2024-11-27T03:39:46,153 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/.tmp/hbase.id 2024-11-27T03:39:46,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741826_1002 (size=42) 2024-11-27T03:39:46,566 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/.tmp/hbase.id]:[hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/hbase.id] 2024-11-27T03:39:46,620 INFO [master/66d3634da959:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T03:39:46,626 INFO [master/66d3634da959:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T03:39:46,649 INFO [master/66d3634da959:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-27T03:39:46,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:46,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:46,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741827_1003 (size=196) 2024-11-27T03:39:47,139 INFO [master/66d3634da959:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:39:47,142 INFO [master/66d3634da959:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T03:39:47,161 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T03:39:47,166 INFO [master/66d3634da959:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T03:39:47,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741828_1004 (size=1189) 2024-11-27T03:39:47,617 INFO [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store 2024-11-27T03:39:47,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741829_1005 (size=34) 2024-11-27T03:39:48,044 INFO [master/66d3634da959:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T03:39:48,048 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:39:48,049 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T03:39:48,050 INFO [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:39:48,050 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:39:48,052 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T03:39:48,052 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:39:48,052 INFO [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:39:48,054 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732678788049Disabling compacts and flushes for region at 1732678788049Disabling writes for close at 1732678788052 (+3 ms)Writing region close event to WAL at 1732678788052Closed at 1732678788052 2024-11-27T03:39:48,056 WARN [master/66d3634da959:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/.initializing 2024-11-27T03:39:48,056 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/WALs/66d3634da959,35691,1732678785028 2024-11-27T03:39:48,065 INFO [master/66d3634da959:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T03:39:48,082 INFO [master/66d3634da959:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=66d3634da959%2C35691%2C1732678785028, suffix=, logDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/WALs/66d3634da959,35691,1732678785028, archiveDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/oldWALs, maxLogs=10 2024-11-27T03:39:48,106 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/WALs/66d3634da959,35691,1732678785028/66d3634da959%2C35691%2C1732678785028.1732678788086, exclude list is [], retry=0 2024-11-27T03:39:48,125 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34855,DS-870dfb75-f227-4af3-bac1-7ad017699541,DISK] 2024-11-27T03:39:48,128 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T03:39:48,168 INFO [master/66d3634da959:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/WALs/66d3634da959,35691,1732678785028/66d3634da959%2C35691%2C1732678785028.1732678788086 2024-11-27T03:39:48,169 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40389:40389)] 2024-11-27T03:39:48,169 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:39:48,170 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:39:48,173 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,174 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,211 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T03:39:48,244 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:48,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:39:48,247 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,251 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T03:39:48,251 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:48,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:39:48,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T03:39:48,257 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:48,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:39:48,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,261 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T03:39:48,262 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:48,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:39:48,263 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,267 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,268 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,273 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,274 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,278 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T03:39:48,282 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T03:39:48,287 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:39:48,289 INFO [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70876798, jitterRate=0.056146591901779175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T03:39:48,297 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732678788187Initializing all the Stores at 1732678788190 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732678788190Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678788191 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678788191Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678788192 (+1 ms)Cleaning up temporary data from old regions at 1732678788274 (+82 ms)Region opened successfully at 1732678788297 (+23 ms) 2024-11-27T03:39:48,299 INFO [master/66d3634da959:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T03:39:48,332 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d70465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=66d3634da959/172.17.0.3:0 2024-11-27T03:39:48,365 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T03:39:48,376 INFO [master/66d3634da959:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T03:39:48,376 INFO [master/66d3634da959:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T03:39:48,379 INFO [master/66d3634da959:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T03:39:48,380 INFO [master/66d3634da959:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T03:39:48,385 INFO [master/66d3634da959:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T03:39:48,385 INFO [master/66d3634da959:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T03:39:48,414 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T03:39:48,425 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T03:39:48,530 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T03:39:48,535 INFO [master/66d3634da959:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T03:39:48,537 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T03:39:48,542 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T03:39:48,545 INFO [master/66d3634da959:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T03:39:48,549 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T03:39:48,562 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T03:39:48,564 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T03:39:48,574 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T03:39:48,594 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T03:39:48,605 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T03:39:48,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T03:39:48,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T03:39:48,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:48,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:48,620 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=66d3634da959,35691,1732678785028, sessionid=0x1017a6010240000, setting cluster-up flag (Was=false) 2024-11-27T03:39:48,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:48,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:48,679 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T03:39:48,681 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=66d3634da959,35691,1732678785028 2024-11-27T03:39:48,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:48,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:39:48,732 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T03:39:48,733 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=66d3634da959,35691,1732678785028 2024-11-27T03:39:48,740 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T03:39:48,798 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(746): ClusterId : 70b3580d-2862-4778-8ac5-3cbc57e5b9a3 2024-11-27T03:39:48,802 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T03:39:48,813 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T03:39:48,818 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T03:39:48,818 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T03:39:48,823 INFO [master/66d3634da959:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T03:39:48,828 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T03:39:48,828 DEBUG [RS:0;66d3634da959:43983 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117b1bee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=66d3634da959/172.17.0.3:0 2024-11-27T03:39:48,830 INFO [master/66d3634da959:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T03:39:48,836 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 66d3634da959,35691,1732678785028 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T03:39:48,844 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/66d3634da959:0, corePoolSize=5, maxPoolSize=5 2024-11-27T03:39:48,844 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/66d3634da959:0, corePoolSize=5, maxPoolSize=5 2024-11-27T03:39:48,844 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/66d3634da959:0, corePoolSize=5, maxPoolSize=5 2024-11-27T03:39:48,845 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/66d3634da959:0, corePoolSize=5, maxPoolSize=5 2024-11-27T03:39:48,845 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/66d3634da959:0, corePoolSize=10, maxPoolSize=10 2024-11-27T03:39:48,845 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:48,845 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/66d3634da959:0, corePoolSize=2, maxPoolSize=2 2024-11-27T03:39:48,845 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;66d3634da959:43983 2024-11-27T03:39:48,846 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:48,847 INFO [master/66d3634da959:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732678818847 2024-11-27T03:39:48,849 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T03:39:48,849 INFO [RS:0;66d3634da959:43983 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T03:39:48,850 INFO [RS:0;66d3634da959:43983 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T03:39:48,850 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T03:39:48,850 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T03:39:48,851 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T03:39:48,851 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T03:39:48,852 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(2659): reportForDuty to master=66d3634da959,35691,1732678785028 with port=43983, startcode=1732678785876 2024-11-27T03:39:48,854 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T03:39:48,854 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T03:39:48,854 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T03:39:48,855 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T03:39:48,858 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:48,858 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T03:39:48,864 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:48,867 DEBUG [RS:0;66d3634da959:43983 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T03:39:48,868 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T03:39:48,870 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T03:39:48,870 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T03:39:48,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741831_1007 (size=1321) 2024-11-27T03:39:48,872 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T03:39:48,872 INFO [master/66d3634da959:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T03:39:48,874 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/66d3634da959:0:becomeActiveMaster-HFileCleaner.large.0-1732678788873,5,FailOnTimeoutGroup] 2024-11-27T03:39:48,888 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/66d3634da959:0:becomeActiveMaster-HFileCleaner.small.0-1732678788874,5,FailOnTimeoutGroup] 2024-11-27T03:39:48,888 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:48,889 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T03:39:48,890 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:48,890 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:48,916 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45747, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T03:39:48,923 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35691 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 66d3634da959,43983,1732678785876 2024-11-27T03:39:48,925 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35691 {}] master.ServerManager(517): Registering regionserver=66d3634da959,43983,1732678785876 2024-11-27T03:39:48,939 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:39:48,940 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35725 2024-11-27T03:39:48,940 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T03:39:48,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T03:39:48,953 DEBUG [RS:0;66d3634da959:43983 {}] zookeeper.ZKUtil(111): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/66d3634da959,43983,1732678785876 2024-11-27T03:39:48,954 WARN [RS:0;66d3634da959:43983 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T03:39:48,954 INFO [RS:0;66d3634da959:43983 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T03:39:48,954 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/WALs/66d3634da959,43983,1732678785876 2024-11-27T03:39:48,956 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [66d3634da959,43983,1732678785876] 2024-11-27T03:39:48,981 INFO [RS:0;66d3634da959:43983 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T03:39:48,998 INFO [RS:0;66d3634da959:43983 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T03:39:49,004 INFO [RS:0;66d3634da959:43983 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T03:39:49,004 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,005 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T03:39:49,012 INFO [RS:0;66d3634da959:43983 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T03:39:49,013 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,013 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,014 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,014 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,014 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,014 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,014 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/66d3634da959:0, corePoolSize=2, maxPoolSize=2 2024-11-27T03:39:49,014 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,014 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,015 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,015 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,015 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,015 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/66d3634da959:0, corePoolSize=1, maxPoolSize=1 2024-11-27T03:39:49,015 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/66d3634da959:0, corePoolSize=3, maxPoolSize=3 2024-11-27T03:39:49,015 DEBUG [RS:0;66d3634da959:43983 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/66d3634da959:0, corePoolSize=3, maxPoolSize=3 2024-11-27T03:39:49,016 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,017 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,017 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,017 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,017 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,017 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,43983,1732678785876-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T03:39:49,038 INFO [RS:0;66d3634da959:43983 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T03:39:49,040 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,43983,1732678785876-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,040 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,040 INFO [RS:0;66d3634da959:43983 {}] regionserver.Replication(171): 66d3634da959,43983,1732678785876 started 2024-11-27T03:39:49,059 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T03:39:49,060 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1482): Serving as 66d3634da959,43983,1732678785876, RpcServer on 66d3634da959/172.17.0.3:43983, sessionid=0x1017a6010240001 2024-11-27T03:39:49,060 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T03:39:49,061 DEBUG [RS:0;66d3634da959:43983 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 66d3634da959,43983,1732678785876 2024-11-27T03:39:49,061 DEBUG [RS:0;66d3634da959:43983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '66d3634da959,43983,1732678785876' 2024-11-27T03:39:49,061 DEBUG [RS:0;66d3634da959:43983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T03:39:49,062 DEBUG [RS:0;66d3634da959:43983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T03:39:49,063 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T03:39:49,063 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T03:39:49,063 DEBUG [RS:0;66d3634da959:43983 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 66d3634da959,43983,1732678785876 2024-11-27T03:39:49,063 DEBUG [RS:0;66d3634da959:43983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '66d3634da959,43983,1732678785876' 2024-11-27T03:39:49,063 DEBUG [RS:0;66d3634da959:43983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T03:39:49,064 DEBUG [RS:0;66d3634da959:43983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T03:39:49,064 DEBUG [RS:0;66d3634da959:43983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T03:39:49,064 INFO [RS:0;66d3634da959:43983 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T03:39:49,065 INFO [RS:0;66d3634da959:43983 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T03:39:49,275 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T03:39:49,276 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:39:49,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741832_1008 (size=32) 2024-11-27T03:39:49,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:39:49,696 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T03:39:49,700 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T03:39:49,700 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:49,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:39:49,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T03:39:49,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T03:39:49,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:49,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:39:49,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T03:39:49,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T03:39:49,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:49,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:39:49,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T03:39:49,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T03:39:49,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:39:49,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:39:49,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T03:39:49,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740 2024-11-27T03:39:49,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740 2024-11-27T03:39:49,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T03:39:49,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T03:39:49,723 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T03:39:49,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T03:39:49,730 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:39:49,731 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73832257, jitterRate=0.10018636286258698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T03:39:49,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732678789692Initializing all the Stores at 1732678789695 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732678789695Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732678789696 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678789696Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732678789696Cleaning up temporary data from old regions at 1732678789722 (+26 ms)Region opened successfully at 1732678789734 (+12 ms) 2024-11-27T03:39:49,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T03:39:49,734 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T03:39:49,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T03:39:49,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T03:39:49,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T03:39:49,736 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T03:39:49,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732678789734Disabling compacts and flushes for region at 1732678789734Disabling writes for close at 1732678789734Writing region close event to WAL at 1732678789735 (+1 ms)Closed at 1732678789736 (+1 ms) 2024-11-27T03:39:49,739 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T03:39:49,739 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T03:39:49,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T03:39:49,754 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T03:39:49,757 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T03:39:49,908 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:50,909 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:51,909 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:52,910 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:53,911 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:54,912 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:55,429 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T03:39:55,429 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T03:39:55,912 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:56,913 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:57,914 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:58,915 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:39:59,915 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:00,916 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:01,917 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:02,918 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:03,918 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:04,919 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:05,920 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:06,920 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:07,921 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:08,922 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:09,923 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:10,923 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:11,924 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:12,925 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:13,925 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:14,038 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T03:40:14,926 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:15,927 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:16,928 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:17,928 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:18,929 WARN [66d3634da959:35691 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T03:40:19,072 INFO [RS:0;66d3634da959:43983 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T03:40:19,076 INFO [RS:0;66d3634da959:43983 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=66d3634da959%2C43983%2C1732678785876, suffix=, logDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/WALs/66d3634da959,43983,1732678785876, archiveDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/oldWALs, maxLogs=32 2024-11-27T03:40:19,094 DEBUG [RS:0;66d3634da959:43983 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/WALs/66d3634da959,43983,1732678785876/66d3634da959%2C43983%2C1732678785876.1732678819079, exclude list is [], retry=0 2024-11-27T03:40:19,098 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34855,DS-870dfb75-f227-4af3-bac1-7ad017699541,DISK] 2024-11-27T03:40:19,102 INFO [RS:0;66d3634da959:43983 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/WALs/66d3634da959,43983,1732678785876/66d3634da959%2C43983%2C1732678785876.1732678819079 2024-11-27T03:40:19,102 DEBUG [RS:0;66d3634da959:43983 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40389:40389)] 2024-11-27T03:40:19,182 DEBUG [66d3634da959:35691 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T03:40:19,195 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:19,202 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 66d3634da959,43983,1732678785876, state=OPENING 2024-11-27T03:40:19,276 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T03:40:19,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:40:19,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:40:19,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T03:40:19,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T03:40:19,291 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T03:40:19,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:19,473 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T03:40:19,477 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T03:40:19,489 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T03:40:19,489 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T03:40:19,490 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T03:40:19,493 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=66d3634da959%2C43983%2C1732678785876.meta, suffix=.meta, logDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/WALs/66d3634da959,43983,1732678785876, archiveDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/oldWALs, maxLogs=32 2024-11-27T03:40:19,510 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/WALs/66d3634da959,43983,1732678785876/66d3634da959%2C43983%2C1732678785876.meta.1732678819496.meta, exclude list is [], retry=0 2024-11-27T03:40:19,515 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34855,DS-870dfb75-f227-4af3-bac1-7ad017699541,DISK] 2024-11-27T03:40:19,519 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/WALs/66d3634da959,43983,1732678785876/66d3634da959%2C43983%2C1732678785876.meta.1732678819496.meta 2024-11-27T03:40:19,519 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40389:40389)] 2024-11-27T03:40:19,520 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:19,522 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T03:40:19,524 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T03:40:19,529 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T03:40:19,534 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T03:40:19,535 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:19,535 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T03:40:19,535 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T03:40:19,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T03:40:19,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T03:40:19,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:19,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:40:19,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T03:40:19,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T03:40:19,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:19,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:40:19,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T03:40:19,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T03:40:19,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:19,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:40:19,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T03:40:19,551 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T03:40:19,551 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:19,552 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T03:40:19,552 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T03:40:19,554 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740 2024-11-27T03:40:19,557 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740 2024-11-27T03:40:19,560 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T03:40:19,560 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T03:40:19,561 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T03:40:19,565 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T03:40:19,567 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70083150, jitterRate=0.04432031512260437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T03:40:19,567 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T03:40:19,569 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732678819536Writing region info on filesystem at 1732678819536Initializing all the Stores at 1732678819538 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732678819538Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732678819538Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678819538Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732678819538Cleaning up temporary data from old regions at 1732678819560 (+22 ms)Running coprocessor post-open hooks at 1732678819567 (+7 ms)Region opened successfully at 1732678819569 (+2 ms) 2024-11-27T03:40:19,577 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732678819464 2024-11-27T03:40:19,591 DEBUG [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T03:40:19,592 INFO [RS_OPEN_META-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T03:40:19,594 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:19,597 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 66d3634da959,43983,1732678785876, state=OPEN 2024-11-27T03:40:19,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T03:40:19,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T03:40:19,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T03:40:19,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T03:40:19,759 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=66d3634da959,43983,1732678785876 2024-11-27T03:40:19,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T03:40:19,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=66d3634da959,43983,1732678785876 in 465 msec 2024-11-27T03:40:19,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T03:40:19,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.0230 sec 2024-11-27T03:40:19,776 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T03:40:19,777 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T03:40:19,796 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T03:40:19,798 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=66d3634da959,43983,1732678785876, seqNum=-1] 2024-11-27T03:40:19,819 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T03:40:19,822 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54619, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T03:40:19,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0740 sec 2024-11-27T03:40:19,847 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732678819846, completionTime=-1 2024-11-27T03:40:19,849 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T03:40:19,849 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T03:40:19,876 INFO [master/66d3634da959:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T03:40:19,877 INFO [master/66d3634da959:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732678879877 2024-11-27T03:40:19,877 INFO [master/66d3634da959:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732678939877 2024-11-27T03:40:19,877 INFO [master/66d3634da959:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-27T03:40:19,879 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,35691,1732678785028-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T03:40:19,879 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,35691,1732678785028-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T03:40:19,879 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,35691,1732678785028-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T03:40:19,880 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-66d3634da959:35691, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T03:40:19,881 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T03:40:19,881 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T03:40:19,907 DEBUG [master/66d3634da959:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T03:40:19,942 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.861sec 2024-11-27T03:40:19,943 INFO [master/66d3634da959:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T03:40:19,946 INFO [master/66d3634da959:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T03:40:19,948 INFO [master/66d3634da959:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T03:40:19,949 INFO [master/66d3634da959:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T03:40:19,949 INFO [master/66d3634da959:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T03:40:19,950 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,35691,1732678785028-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T03:40:19,950 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,35691,1732678785028-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T03:40:19,959 DEBUG [master/66d3634da959:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T03:40:19,960 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T03:40:19,960 INFO [master/66d3634da959:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66d3634da959,35691,1732678785028-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T03:40:20,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb9f78e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T03:40:20,039 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T03:40:20,039 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T03:40:20,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 66d3634da959,35691,-1 for getting cluster id 2024-11-27T03:40:20,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T03:40:20,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '70b3580d-2862-4778-8ac5-3cbc57e5b9a3' 2024-11-27T03:40:20,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T03:40:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "70b3580d-2862-4778-8ac5-3cbc57e5b9a3" 2024-11-27T03:40:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcb2cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T03:40:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [66d3634da959,35691,-1] 2024-11-27T03:40:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T03:40:20,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:40:20,065 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T03:40:20,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2037c9b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T03:40:20,069 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T03:40:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=66d3634da959,43983,1732678785876, seqNum=-1] 2024-11-27T03:40:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T03:40:20,078 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T03:40:20,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=66d3634da959,35691,1732678785028 2024-11-27T03:40:20,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14379079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T03:40:20,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 66d3634da959,35691,-1 for getting cluster id 2024-11-27T03:40:20,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T03:40:20,102 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '70b3580d-2862-4778-8ac5-3cbc57e5b9a3' 2024-11-27T03:40:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T03:40:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "70b3580d-2862-4778-8ac5-3cbc57e5b9a3" 2024-11-27T03:40:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5082d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T03:40:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [66d3634da959,35691,-1] 2024-11-27T03:40:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T03:40:20,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:40:20,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T03:40:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad2d8e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T03:40:20,121 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=239, ProcessCount=11, AvailableMemoryMB=8162 2024-11-27T03:40:20,128 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T03:40:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 66d3634da959,35691,1732678785028 2024-11-27T03:40:20,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a1132fb 2024-11-27T03:40:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T03:40:20,141 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T03:40:20,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:40:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-27T03:40:20,164 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:40:20,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-27T03:40:20,167 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:20,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:40:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T03:40:20,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741835_1011 (size=428) 2024-11-27T03:40:20,587 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => da5e2d6a51e95d62df0d394d31d3005f, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:40:20,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741836_1012 (size=79) 2024-11-27T03:40:20,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:20,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing da5e2d6a51e95d62df0d394d31d3005f, disabling compactions & flushes 2024-11-27T03:40:20,599 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:40:20,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:40:20,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. after waiting 0 ms 2024-11-27T03:40:20,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:40:20,600 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:40:20,600 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for da5e2d6a51e95d62df0d394d31d3005f: Waiting for close lock at 1732678820599Disabling compacts and flushes for region at 1732678820599Disabling writes for close at 1732678820599Writing region close event to WAL at 1732678820600 (+1 ms)Closed at 1732678820600 2024-11-27T03:40:20,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:40:20,609 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732678820602"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678820602"}]},"ts":"1732678820602"} 2024-11-27T03:40:20,614 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:40:20,616 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:40:20,619 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678820617"}]},"ts":"1732678820617"} 2024-11-27T03:40:20,625 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-27T03:40:20,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=da5e2d6a51e95d62df0d394d31d3005f, ASSIGN}] 2024-11-27T03:40:20,631 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=da5e2d6a51e95d62df0d394d31d3005f, ASSIGN 2024-11-27T03:40:20,633 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=da5e2d6a51e95d62df0d394d31d3005f, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:40:20,785 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=da5e2d6a51e95d62df0d394d31d3005f, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:20,792 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=da5e2d6a51e95d62df0d394d31d3005f, ASSIGN because future has completed 2024-11-27T03:40:20,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5e2d6a51e95d62df0d394d31d3005f, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:20,957 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:40:20,957 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => da5e2d6a51e95d62df0d394d31d3005f, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:20,958 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,958 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:20,958 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,958 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,962 INFO [StoreOpener-da5e2d6a51e95d62df0d394d31d3005f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,964 INFO [StoreOpener-da5e2d6a51e95d62df0d394d31d3005f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da5e2d6a51e95d62df0d394d31d3005f columnFamilyName testFamily 2024-11-27T03:40:20,965 DEBUG [StoreOpener-da5e2d6a51e95d62df0d394d31d3005f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:20,966 INFO [StoreOpener-da5e2d6a51e95d62df0d394d31d3005f-1 {}] regionserver.HStore(327): Store=da5e2d6a51e95d62df0d394d31d3005f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:40:20,966 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,968 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,969 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,970 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,970 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,974 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,978 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:40:20,979 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened da5e2d6a51e95d62df0d394d31d3005f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61064437, jitterRate=-0.09006898105144501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:40:20,979 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:40:20,980 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for da5e2d6a51e95d62df0d394d31d3005f: Running coprocessor pre-open hook at 1732678820959Writing region info on filesystem at 1732678820959Initializing all the Stores at 1732678820961 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678820961Cleaning up temporary data from old regions at 1732678820970 (+9 ms)Running coprocessor post-open hooks at 1732678820979 (+9 ms)Region opened successfully at 1732678820980 (+1 ms) 2024-11-27T03:40:20,983 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f., pid=6, masterSystemTime=1732678820949 2024-11-27T03:40:20,987 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:40:20,987 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:40:20,988 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=da5e2d6a51e95d62df0d394d31d3005f, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:20,993 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure da5e2d6a51e95d62df0d394d31d3005f, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:40:21,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T03:40:21,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure da5e2d6a51e95d62df0d394d31d3005f, server=66d3634da959,43983,1732678785876 in 201 msec 2024-11-27T03:40:21,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T03:40:21,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=da5e2d6a51e95d62df0d394d31d3005f, ASSIGN in 374 msec 2024-11-27T03:40:21,008 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:40:21,008 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678821008"}]},"ts":"1732678821008"} 2024-11-27T03:40:21,012 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-27T03:40:21,014 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:40:21,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 860 msec 2024-11-27T03:40:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T03:40:21,184 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-27T03:40:21,184 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-27T03:40:21,186 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:21,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-27T03:40:21,196 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:21,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-27T03:40:21,200 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T03:40:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=66d3634da959,43983,1732678785876, seqNum=-1] 2024-11-27T03:40:21,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T03:40:21,205 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T03:40:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:21,228 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:40:21,232 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-27T03:40:21,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-27T03:40:21,243 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-27T03:40:21,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-27T03:40:21,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6384422693130731528 2024-11-27T03:40:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678824245, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:21,278 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-27T03:40:21,280 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-27T03:40:21,281 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:21,284 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:21,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6384422693130731530 2024-11-27T03:40:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678824284, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:21,290 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:21,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6384422693130731531 2024-11-27T03:40:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678824290, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:21,296 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:21,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6384422693130731532 2024-11-27T03:40:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678824296, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:21,302 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:21,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/66d3634da959:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365225140_22 at /127.0.0.1:55292 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 239), ProcessCount=11 (was 11), AvailableMemoryMB=8144 (was 8162) 2024-11-27T03:40:21,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=8142 2024-11-27T03:40:21,330 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:40:21,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-27T03:40:21,335 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:40:21,336 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:21,336 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-27T03:40:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T03:40:21,338 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:40:21,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741837_1013 (size=414) 2024-11-27T03:40:21,751 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e2ec5a720546e890339825e7278cf91d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:40:21,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741838_1014 (size=65) 2024-11-27T03:40:22,162 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:22,162 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing e2ec5a720546e890339825e7278cf91d, disabling compactions & flushes 2024-11-27T03:40:22,162 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:40:22,162 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:40:22,162 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. after waiting 0 ms 2024-11-27T03:40:22,162 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:40:22,162 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:40:22,163 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for e2ec5a720546e890339825e7278cf91d: Waiting for close lock at 1732678822162Disabling compacts and flushes for region at 1732678822162Disabling writes for close at 1732678822162Writing region close event to WAL at 1732678822162Closed at 1732678822162 2024-11-27T03:40:22,165 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:40:22,165 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732678822165"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678822165"}]},"ts":"1732678822165"} 2024-11-27T03:40:22,169 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:40:22,171 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:40:22,171 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678822171"}]},"ts":"1732678822171"} 2024-11-27T03:40:22,174 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-27T03:40:22,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e2ec5a720546e890339825e7278cf91d, ASSIGN}] 2024-11-27T03:40:22,178 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e2ec5a720546e890339825e7278cf91d, ASSIGN 2024-11-27T03:40:22,180 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e2ec5a720546e890339825e7278cf91d, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:40:22,331 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e2ec5a720546e890339825e7278cf91d, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:22,336 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e2ec5a720546e890339825e7278cf91d, ASSIGN because future has completed 2024-11-27T03:40:22,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2ec5a720546e890339825e7278cf91d, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T03:40:22,502 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:40:22,502 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e2ec5a720546e890339825e7278cf91d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:22,503 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,503 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:22,503 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,503 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,505 INFO [StoreOpener-e2ec5a720546e890339825e7278cf91d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,508 INFO [StoreOpener-e2ec5a720546e890339825e7278cf91d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2ec5a720546e890339825e7278cf91d columnFamilyName testFamily 2024-11-27T03:40:22,508 DEBUG [StoreOpener-e2ec5a720546e890339825e7278cf91d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:22,509 INFO [StoreOpener-e2ec5a720546e890339825e7278cf91d-1 {}] regionserver.HStore(327): Store=e2ec5a720546e890339825e7278cf91d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:40:22,509 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,510 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,511 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,512 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,512 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,515 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,518 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:40:22,519 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened e2ec5a720546e890339825e7278cf91d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59676315, jitterRate=-0.1107536107301712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:40:22,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:40:22,520 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e2ec5a720546e890339825e7278cf91d: Running coprocessor pre-open hook at 1732678822503Writing region info on filesystem at 1732678822503Initializing all the Stores at 1732678822505 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678822505Cleaning up temporary data from old regions at 1732678822512 (+7 ms)Running coprocessor post-open hooks at 1732678822519 (+7 ms)Region opened successfully at 1732678822520 (+1 ms) 2024-11-27T03:40:22,522 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., pid=9, masterSystemTime=1732678822493 2024-11-27T03:40:22,526 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:40:22,526 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:40:22,527 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e2ec5a720546e890339825e7278cf91d, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:22,531 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2ec5a720546e890339825e7278cf91d, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:40:22,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-27T03:40:22,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e2ec5a720546e890339825e7278cf91d, server=66d3634da959,43983,1732678785876 in 197 msec 2024-11-27T03:40:22,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T03:40:22,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e2ec5a720546e890339825e7278cf91d, ASSIGN in 364 msec 2024-11-27T03:40:22,546 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:40:22,547 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678822546"}]},"ts":"1732678822546"} 2024-11-27T03:40:22,550 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-27T03:40:22,552 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:40:22,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2210 sec 2024-11-27T03:40:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T03:40:24,384 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-27T03:40:24,384 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-27T03:40:24,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:24,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-27T03:40:24,390 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:24,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-27T03:40:24,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:24,410 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:40:24,410 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-27T03:40:24,415 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T03:40:24,417 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-27T03:40:25,078 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:40:25,114 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-27T03:40:25,117 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T03:40:25,118 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-27T03:40:25,428 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T03:40:25,429 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T03:40:25,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T03:40:25,430 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T03:40:25,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-27T03:40:25,431 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T03:40:25,432 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-27T03:40:25,432 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-27T03:40:30,430 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., hostname=66d3634da959,43983,1732678785876, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., hostname=66d3634da959,43983,1732678785876, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=66d3634da959:43983 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-27T03:40:30,431 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., hostname=66d3634da959,43983,1732678785876, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-27T03:40:30,431 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., hostname=66d3634da959,43983,1732678785876, seqNum=2 from cache 2024-11-27T03:40:30,441 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=257 (was 227) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8107 (was 8142) 2024-11-27T03:40:30,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=257, ProcessCount=11, AvailableMemoryMB=8107 2024-11-27T03:40:30,453 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:40:30,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-27T03:40:30,457 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:40:30,457 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:30,458 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-27T03:40:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T03:40:30,460 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:40:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741839_1015 (size=412) 2024-11-27T03:40:30,872 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3fc02201540a0ca11e341f69086bc8a6, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:40:30,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741840_1016 (size=63) 2024-11-27T03:40:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-27T03:40:31,284 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:31,284 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 3fc02201540a0ca11e341f69086bc8a6, disabling compactions & flushes 2024-11-27T03:40:31,284 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:40:31,285 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:40:31,285 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. after waiting 0 ms 2024-11-27T03:40:31,285 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:40:31,285 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:40:31,285 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3fc02201540a0ca11e341f69086bc8a6: Waiting for close lock at 1732678831284Disabling compacts and flushes for region at 1732678831284Disabling writes for close at 1732678831285 (+1 ms)Writing region close event to WAL at 1732678831285Closed at 1732678831285 2024-11-27T03:40:31,288 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:40:31,288 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732678831288"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678831288"}]},"ts":"1732678831288"} 2024-11-27T03:40:31,293 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:40:31,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:40:31,295 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678831294"}]},"ts":"1732678831294"} 2024-11-27T03:40:31,298 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-27T03:40:31,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=3fc02201540a0ca11e341f69086bc8a6, ASSIGN}] 2024-11-27T03:40:31,301 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=3fc02201540a0ca11e341f69086bc8a6, ASSIGN 2024-11-27T03:40:31,303 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=3fc02201540a0ca11e341f69086bc8a6, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:40:31,454 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3fc02201540a0ca11e341f69086bc8a6, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:31,459 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=3fc02201540a0ca11e341f69086bc8a6, ASSIGN because future has completed 2024-11-27T03:40:31,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fc02201540a0ca11e341f69086bc8a6, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T03:40:31,622 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:40:31,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fc02201540a0ca11e341f69086bc8a6, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:31,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:31,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,624 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,626 INFO [StoreOpener-3fc02201540a0ca11e341f69086bc8a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,629 INFO [StoreOpener-3fc02201540a0ca11e341f69086bc8a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fc02201540a0ca11e341f69086bc8a6 columnFamilyName testFamily 2024-11-27T03:40:31,629 DEBUG [StoreOpener-3fc02201540a0ca11e341f69086bc8a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:31,630 INFO [StoreOpener-3fc02201540a0ca11e341f69086bc8a6-1 {}] regionserver.HStore(327): Store=3fc02201540a0ca11e341f69086bc8a6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:40:31,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,632 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,633 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,634 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,634 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,638 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:40:31,643 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 3fc02201540a0ca11e341f69086bc8a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59069814, jitterRate=-0.119791179895401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:40:31,643 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:40:31,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3fc02201540a0ca11e341f69086bc8a6: Running coprocessor pre-open hook at 1732678831624Writing region info on filesystem at 1732678831624Initializing all the Stores at 1732678831626 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678831626Cleaning up temporary data from old regions at 1732678831634 (+8 ms)Running coprocessor post-open hooks at 1732678831643 (+9 ms)Region opened successfully at 1732678831644 (+1 ms) 2024-11-27T03:40:31,645 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6., pid=12, masterSystemTime=1732678831615 2024-11-27T03:40:31,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:40:31,649 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:40:31,650 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3fc02201540a0ca11e341f69086bc8a6, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:31,653 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fc02201540a0ca11e341f69086bc8a6, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:40:31,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T03:40:31,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3fc02201540a0ca11e341f69086bc8a6, server=66d3634da959,43983,1732678785876 in 195 msec 2024-11-27T03:40:31,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T03:40:31,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=3fc02201540a0ca11e341f69086bc8a6, ASSIGN in 362 msec 2024-11-27T03:40:31,666 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:40:31,667 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678831666"}]},"ts":"1732678831666"} 2024-11-27T03:40:31,670 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-27T03:40:31,672 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:40:31,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2190 sec 2024-11-27T03:40:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T03:40:33,484 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-27T03:40:33,484 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-27T03:40:33,485 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:33,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-27T03:40:33,493 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:33,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-27T03:40:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:33,510 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:40:33,510 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-27T03:40:33,514 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T03:40:33,517 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T03:40:33,519 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T03:40:33,520 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-27T03:40:35,428 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-27T03:40:35,429 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T03:40:37,153 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:40:37,182 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-27T03:40:39,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731541' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T03:40:39,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:44326 deadline: 1732678845523, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731541' does not exist 2024-11-27T03:40:39,531 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 66d3634da959,43983,1732678785876 for closing scanner id = 6384422693130731541 for 3fc02201540a0ca11e341f69086bc8a6 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731541' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T03:40:39,533 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 257), ProcessCount=11 (was 11), AvailableMemoryMB=8093 (was 8107) 2024-11-27T03:40:39,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=241, ProcessCount=11, AvailableMemoryMB=8093 2024-11-27T03:40:39,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:40:39,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-27T03:40:39,548 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:40:39,548 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:39,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-27T03:40:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T03:40:39,549 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:40:39,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741841_1017 (size=423) 2024-11-27T03:40:39,961 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c51a4c2e130a00f12e17466aa3cb9eb0, NAME => 'testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:40:39,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741842_1018 (size=74) 2024-11-27T03:40:40,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-27T03:40:40,371 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:40,371 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing c51a4c2e130a00f12e17466aa3cb9eb0, disabling compactions & flushes 2024-11-27T03:40:40,371 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:40:40,371 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:40:40,371 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. after waiting 0 ms 2024-11-27T03:40:40,371 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:40:40,371 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:40:40,372 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for c51a4c2e130a00f12e17466aa3cb9eb0: Waiting for close lock at 1732678840371Disabling compacts and flushes for region at 1732678840371Disabling writes for close at 1732678840371Writing region close event to WAL at 1732678840371Closed at 1732678840371 2024-11-27T03:40:40,374 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:40:40,375 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732678840374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678840374"}]},"ts":"1732678840374"} 2024-11-27T03:40:40,379 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:40:40,381 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:40:40,381 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678840381"}]},"ts":"1732678840381"} 2024-11-27T03:40:40,384 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-27T03:40:40,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=c51a4c2e130a00f12e17466aa3cb9eb0, ASSIGN}] 2024-11-27T03:40:40,387 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=c51a4c2e130a00f12e17466aa3cb9eb0, ASSIGN 2024-11-27T03:40:40,389 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=c51a4c2e130a00f12e17466aa3cb9eb0, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:40:40,539 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c51a4c2e130a00f12e17466aa3cb9eb0, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:40,543 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=c51a4c2e130a00f12e17466aa3cb9eb0, ASSIGN because future has completed 2024-11-27T03:40:40,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c51a4c2e130a00f12e17466aa3cb9eb0, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T03:40:40,704 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:40:40,705 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => c51a4c2e130a00f12e17466aa3cb9eb0, NAME => 'testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:40,705 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,705 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:40,706 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,706 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,709 INFO [StoreOpener-c51a4c2e130a00f12e17466aa3cb9eb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,713 INFO [StoreOpener-c51a4c2e130a00f12e17466aa3cb9eb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c51a4c2e130a00f12e17466aa3cb9eb0 columnFamilyName testFamily 2024-11-27T03:40:40,713 DEBUG [StoreOpener-c51a4c2e130a00f12e17466aa3cb9eb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:40,714 INFO [StoreOpener-c51a4c2e130a00f12e17466aa3cb9eb0-1 {}] regionserver.HStore(327): Store=c51a4c2e130a00f12e17466aa3cb9eb0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:40:40,714 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,716 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,716 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,717 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,717 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,720 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,724 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:40:40,725 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened c51a4c2e130a00f12e17466aa3cb9eb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65619896, jitterRate=-0.022187352180480957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:40:40,725 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:40:40,726 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for c51a4c2e130a00f12e17466aa3cb9eb0: Running coprocessor pre-open hook at 1732678840706Writing region info on filesystem at 1732678840706Initializing all the Stores at 1732678840708 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678840708Cleaning up temporary data from old regions at 1732678840717 (+9 ms)Running coprocessor post-open hooks at 1732678840725 (+8 ms)Region opened successfully at 1732678840726 (+1 ms) 2024-11-27T03:40:40,727 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0., pid=15, masterSystemTime=1732678840697 2024-11-27T03:40:40,731 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:40:40,731 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:40:40,732 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=c51a4c2e130a00f12e17466aa3cb9eb0, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:40,735 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure c51a4c2e130a00f12e17466aa3cb9eb0, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:40:40,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T03:40:40,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure c51a4c2e130a00f12e17466aa3cb9eb0, server=66d3634da959,43983,1732678785876 in 193 msec 2024-11-27T03:40:40,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T03:40:40,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=c51a4c2e130a00f12e17466aa3cb9eb0, ASSIGN in 356 msec 2024-11-27T03:40:40,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:40:40,747 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678840746"}]},"ts":"1732678840746"} 2024-11-27T03:40:40,750 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-27T03:40:40,751 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:40:40,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2070 sec 2024-11-27T03:40:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T03:40:42,584 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-27T03:40:42,584 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-27T03:40:42,585 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:42,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-27T03:40:42,593 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:42,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-27T03:40:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:42,612 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:40:42,612 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-27T03:40:42,616 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-27T03:40:42,619 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-27T03:40:42,620 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-27T03:40:42,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6384422693130731545 2024-11-27T03:40:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678845620, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:42,625 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-27T03:40:42,626 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-27T03:40:42,627 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:42,628 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:42,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6384422693130731547 2024-11-27T03:40:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678845628, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:42,632 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:42,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6384422693130731548 2024-11-27T03:40:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678845631, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:42,636 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:42,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6384422693130731549 2024-11-27T03:40:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:44326 deadline: 1732678845635, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T03:40:42,640 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T03:40:42,650 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-365225140_22 at /127.0.0.1:44652 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 241), ProcessCount=11 (was 11), AvailableMemoryMB=8087 (was 8093) 2024-11-27T03:40:42,661 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=8086 2024-11-27T03:40:42,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:40:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-27T03:40:42,667 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:40:42,667 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:42,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-27T03:40:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T03:40:42,669 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:40:42,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741843_1019 (size=418) 2024-11-27T03:40:43,082 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f4cc431e1f47cb481fdc44a56ad6342, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:40:43,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741844_1020 (size=69) 2024-11-27T03:40:43,183 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-27T03:40:43,492 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:43,492 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 8f4cc431e1f47cb481fdc44a56ad6342, disabling compactions & flushes 2024-11-27T03:40:43,492 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:40:43,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:40:43,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. after waiting 0 ms 2024-11-27T03:40:43,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:40:43,493 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:40:43,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f4cc431e1f47cb481fdc44a56ad6342: Waiting for close lock at 1732678843492Disabling compacts and flushes for region at 1732678843492Disabling writes for close at 1732678843493 (+1 ms)Writing region close event to WAL at 1732678843493Closed at 1732678843493 2024-11-27T03:40:43,494 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:40:43,495 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732678843495"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678843495"}]},"ts":"1732678843495"} 2024-11-27T03:40:43,498 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:40:43,500 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:40:43,500 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678843500"}]},"ts":"1732678843500"} 2024-11-27T03:40:43,504 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-27T03:40:43,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8f4cc431e1f47cb481fdc44a56ad6342, ASSIGN}] 2024-11-27T03:40:43,506 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8f4cc431e1f47cb481fdc44a56ad6342, ASSIGN 2024-11-27T03:40:43,508 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8f4cc431e1f47cb481fdc44a56ad6342, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:40:43,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8f4cc431e1f47cb481fdc44a56ad6342, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:43,662 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8f4cc431e1f47cb481fdc44a56ad6342, ASSIGN because future has completed 2024-11-27T03:40:43,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f4cc431e1f47cb481fdc44a56ad6342, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T03:40:43,822 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:40:43,822 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f4cc431e1f47cb481fdc44a56ad6342, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:43,823 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,823 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:43,823 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,823 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,826 INFO [StoreOpener-8f4cc431e1f47cb481fdc44a56ad6342-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,828 INFO [StoreOpener-8f4cc431e1f47cb481fdc44a56ad6342-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f4cc431e1f47cb481fdc44a56ad6342 columnFamilyName testFamily 2024-11-27T03:40:43,829 DEBUG [StoreOpener-8f4cc431e1f47cb481fdc44a56ad6342-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:43,830 INFO [StoreOpener-8f4cc431e1f47cb481fdc44a56ad6342-1 {}] regionserver.HStore(327): Store=8f4cc431e1f47cb481fdc44a56ad6342/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:40:43,830 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,831 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,832 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,833 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,833 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,835 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,839 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:40:43,839 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 8f4cc431e1f47cb481fdc44a56ad6342; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67827732, jitterRate=0.010711967945098877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:40:43,840 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:40:43,841 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 8f4cc431e1f47cb481fdc44a56ad6342: Running coprocessor pre-open hook at 1732678843823Writing region info on filesystem at 1732678843823Initializing all the Stores at 1732678843825 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678843825Cleaning up temporary data from old regions at 1732678843833 (+8 ms)Running coprocessor post-open hooks at 1732678843840 (+7 ms)Region opened successfully at 1732678843841 (+1 ms) 2024-11-27T03:40:43,842 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., pid=18, masterSystemTime=1732678843816 2024-11-27T03:40:43,846 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:40:43,846 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:40:43,847 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8f4cc431e1f47cb481fdc44a56ad6342, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:43,850 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f4cc431e1f47cb481fdc44a56ad6342, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:40:43,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T03:40:43,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f4cc431e1f47cb481fdc44a56ad6342, server=66d3634da959,43983,1732678785876 in 189 msec 2024-11-27T03:40:43,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-27T03:40:43,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8f4cc431e1f47cb481fdc44a56ad6342, ASSIGN in 351 msec 2024-11-27T03:40:43,860 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:40:43,861 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678843861"}]},"ts":"1732678843861"} 2024-11-27T03:40:43,864 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-27T03:40:43,865 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:40:43,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2030 sec 2024-11-27T03:40:44,039 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T03:40:45,428 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-27T03:40:45,429 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-27T03:40:45,430 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-27T03:40:45,430 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-27T03:40:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T03:40:45,704 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-27T03:40:45,704 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-27T03:40:45,704 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:45,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-27T03:40:45,712 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:45,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-27T03:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:45,728 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:40:45,729 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-27T03:40:45,732 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T03:40:45,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T03:40:46,209 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:40:46,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=66d3634da959:43983 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-27T03:40:46,740 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-27T03:40:46,740 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2 from cache 2024-11-27T03:40:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-27T03:40:47,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:47,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T03:40:48,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=66d3634da959:43983 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=982ms, rpcTimeout=976ms 2024-11-27T03:40:48,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=982ms, rpcTimeout=976ms 2024-11-27T03:40:48,739 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., hostname=66d3634da959,43983,1732678785876, seqNum=2 from cache 2024-11-27T03:40:48,751 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 226), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=8083 (was 8086) 2024-11-27T03:40:48,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=8082 2024-11-27T03:40:48,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:40:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-27T03:40:48,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:40:48,769 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:48,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-27T03:40:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T03:40:48,771 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:40:48,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741845_1021 (size=416) 2024-11-27T03:40:49,183 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2ed84224f9a878accc61e9c054395e36, NAME => 'testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:40:49,185 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-27T03:40:49,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741846_1022 (size=67) 2024-11-27T03:40:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-27T03:40:49,593 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:49,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 2ed84224f9a878accc61e9c054395e36, disabling compactions & flushes 2024-11-27T03:40:49,594 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:40:49,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:40:49,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. after waiting 0 ms 2024-11-27T03:40:49,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:40:49,594 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:40:49,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2ed84224f9a878accc61e9c054395e36: Waiting for close lock at 1732678849594Disabling compacts and flushes for region at 1732678849594Disabling writes for close at 1732678849594Writing region close event to WAL at 1732678849594Closed at 1732678849594 2024-11-27T03:40:49,597 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:40:49,598 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732678849598"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678849598"}]},"ts":"1732678849598"} 2024-11-27T03:40:49,602 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:40:49,603 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:40:49,603 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678849603"}]},"ts":"1732678849603"} 2024-11-27T03:40:49,606 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-27T03:40:49,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2ed84224f9a878accc61e9c054395e36, ASSIGN}] 2024-11-27T03:40:49,608 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2ed84224f9a878accc61e9c054395e36, ASSIGN 2024-11-27T03:40:49,610 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2ed84224f9a878accc61e9c054395e36, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:40:49,761 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=2ed84224f9a878accc61e9c054395e36, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:49,764 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2ed84224f9a878accc61e9c054395e36, ASSIGN because future has completed 2024-11-27T03:40:49,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ed84224f9a878accc61e9c054395e36, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T03:40:49,926 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:40:49,927 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ed84224f9a878accc61e9c054395e36, NAME => 'testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:49,927 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,928 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:49,928 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,928 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,930 INFO [StoreOpener-2ed84224f9a878accc61e9c054395e36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,932 INFO [StoreOpener-2ed84224f9a878accc61e9c054395e36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ed84224f9a878accc61e9c054395e36 columnFamilyName testFamily 2024-11-27T03:40:49,932 DEBUG [StoreOpener-2ed84224f9a878accc61e9c054395e36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:49,933 INFO [StoreOpener-2ed84224f9a878accc61e9c054395e36-1 {}] regionserver.HStore(327): Store=2ed84224f9a878accc61e9c054395e36/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:40:49,933 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,934 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,934 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,935 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,935 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,937 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,941 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:40:49,942 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 2ed84224f9a878accc61e9c054395e36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72033745, jitterRate=0.07338644564151764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:40:49,942 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:40:49,943 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 2ed84224f9a878accc61e9c054395e36: Running coprocessor pre-open hook at 1732678849928Writing region info on filesystem at 1732678849928Initializing all the Stores at 1732678849929 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678849929Cleaning up temporary data from old regions at 1732678849935 (+6 ms)Running coprocessor post-open hooks at 1732678849942 (+7 ms)Region opened successfully at 1732678849942 2024-11-27T03:40:49,944 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36., pid=21, masterSystemTime=1732678849917 2024-11-27T03:40:49,947 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:40:49,947 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:40:49,948 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=2ed84224f9a878accc61e9c054395e36, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:49,951 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ed84224f9a878accc61e9c054395e36, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:40:49,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-27T03:40:49,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ed84224f9a878accc61e9c054395e36, server=66d3634da959,43983,1732678785876 in 189 msec 2024-11-27T03:40:49,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T03:40:49,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2ed84224f9a878accc61e9c054395e36, ASSIGN in 350 msec 2024-11-27T03:40:49,961 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:40:49,962 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678849961"}]},"ts":"1732678849961"} 2024-11-27T03:40:49,965 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-27T03:40:49,966 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:40:49,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2010 sec 2024-11-27T03:40:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T03:40:51,804 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-27T03:40:51,804 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-27T03:40:51,805 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:51,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-27T03:40:51,813 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:51,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-27T03:40:51,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:51,830 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:40:51,831 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-27T03:40:51,834 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T03:40:51,836 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T03:40:51,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T03:40:51,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-27T03:40:54,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731560' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T03:40:54,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:44326 deadline: 1732678855842, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731560' does not exist 2024-11-27T03:40:54,843 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 66d3634da959,43983,1732678785876 for closing scanner id = 6384422693130731560 for 2ed84224f9a878accc61e9c054395e36 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731560' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T03:40:54,849 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=225 (was 227), ProcessCount=11 (was 11), AvailableMemoryMB=8070 (was 8082) 2024-11-27T03:40:54,857 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=225, ProcessCount=11, AvailableMemoryMB=8070 2024-11-27T03:40:54,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:40:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-27T03:40:54,864 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:40:54,864 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:54,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-27T03:40:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T03:40:54,868 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:40:54,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741847_1023 (size=411) 2024-11-27T03:40:55,286 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e28f02d18ffad4f680bcdc3c36dbe311, NAME => 'testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:40:55,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741848_1024 (size=62) 2024-11-27T03:40:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-27T03:40:55,432 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:40:55,462 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-27T03:40:55,462 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-27T03:40:55,462 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T03:40:55,697 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:55,697 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing e28f02d18ffad4f680bcdc3c36dbe311, disabling compactions & flushes 2024-11-27T03:40:55,697 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:40:55,697 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:40:55,697 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. after waiting 0 ms 2024-11-27T03:40:55,697 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:40:55,698 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:40:55,698 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for e28f02d18ffad4f680bcdc3c36dbe311: Waiting for close lock at 1732678855697Disabling compacts and flushes for region at 1732678855697Disabling writes for close at 1732678855697Writing region close event to WAL at 1732678855697Closed at 1732678855697 2024-11-27T03:40:55,699 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:40:55,699 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732678855699"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678855699"}]},"ts":"1732678855699"} 2024-11-27T03:40:55,702 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:40:55,704 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:40:55,704 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678855704"}]},"ts":"1732678855704"} 2024-11-27T03:40:55,707 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-27T03:40:55,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e28f02d18ffad4f680bcdc3c36dbe311, ASSIGN}] 2024-11-27T03:40:55,709 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e28f02d18ffad4f680bcdc3c36dbe311, ASSIGN 2024-11-27T03:40:55,711 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e28f02d18ffad4f680bcdc3c36dbe311, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:40:55,862 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e28f02d18ffad4f680bcdc3c36dbe311, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:55,865 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e28f02d18ffad4f680bcdc3c36dbe311, ASSIGN because future has completed 2024-11-27T03:40:55,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e28f02d18ffad4f680bcdc3c36dbe311, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:40:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T03:40:56,024 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:40:56,025 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => e28f02d18ffad4f680bcdc3c36dbe311, NAME => 'testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:40:56,025 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,025 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:40:56,025 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,025 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,028 INFO [StoreOpener-e28f02d18ffad4f680bcdc3c36dbe311-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,030 INFO [StoreOpener-e28f02d18ffad4f680bcdc3c36dbe311-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e28f02d18ffad4f680bcdc3c36dbe311 columnFamilyName testFamily 2024-11-27T03:40:56,031 DEBUG [StoreOpener-e28f02d18ffad4f680bcdc3c36dbe311-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:40:56,031 INFO [StoreOpener-e28f02d18ffad4f680bcdc3c36dbe311-1 {}] regionserver.HStore(327): Store=e28f02d18ffad4f680bcdc3c36dbe311/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:40:56,031 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,033 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,033 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,034 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,034 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,036 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,039 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:40:56,040 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened e28f02d18ffad4f680bcdc3c36dbe311; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58723245, jitterRate=-0.12495546042919159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:40:56,040 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:40:56,041 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for e28f02d18ffad4f680bcdc3c36dbe311: Running coprocessor pre-open hook at 1732678856026Writing region info on filesystem at 1732678856026Initializing all the Stores at 1732678856027 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678856027Cleaning up temporary data from old regions at 1732678856034 (+7 ms)Running coprocessor post-open hooks at 1732678856040 (+6 ms)Region opened successfully at 1732678856041 (+1 ms) 2024-11-27T03:40:56,042 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311., pid=24, masterSystemTime=1732678856019 2024-11-27T03:40:56,045 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:40:56,045 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:40:56,046 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e28f02d18ffad4f680bcdc3c36dbe311, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:40:56,049 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e28f02d18ffad4f680bcdc3c36dbe311, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:40:56,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T03:40:56,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure e28f02d18ffad4f680bcdc3c36dbe311, server=66d3634da959,43983,1732678785876 in 186 msec 2024-11-27T03:40:56,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-27T03:40:56,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e28f02d18ffad4f680bcdc3c36dbe311, ASSIGN in 347 msec 2024-11-27T03:40:56,060 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:40:56,060 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678856060"}]},"ts":"1732678856060"} 2024-11-27T03:40:56,063 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-27T03:40:56,065 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:40:56,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2050 sec 2024-11-27T03:40:57,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T03:40:57,906 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-27T03:40:57,906 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-27T03:40:57,907 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:57,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-27T03:40:57,915 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:40:57,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-27T03:40:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:40:57,928 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:40:57,928 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-27T03:40:57,931 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T03:40:57,933 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T03:40:57,935 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T03:40:57,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-27T03:40:59,963 INFO [master/66d3634da959:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T03:40:59,963 INFO [master/66d3634da959:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T03:41:00,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731564' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T03:41:00,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:44326 deadline: 1732678861947, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731564' does not exist 2024-11-27T03:41:00,951 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 66d3634da959,43983,1732678785876 for closing scanner id = 6384422693130731564 for e28f02d18ffad4f680bcdc3c36dbe311 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731564' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T03:41:00,956 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=212 (was 225), ProcessCount=11 (was 11), AvailableMemoryMB=8040 (was 8070) 2024-11-27T03:41:00,965 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=212, ProcessCount=11, AvailableMemoryMB=8040 2024-11-27T03:41:00,968 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:41:00,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-27T03:41:00,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:41:00,972 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:41:00,972 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-27T03:41:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T03:41:00,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:41:00,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741849_1025 (size=419) 2024-11-27T03:41:01,384 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a2c4046a8362f1943854d807513f4b6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:41:01,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741850_1026 (size=70) 2024-11-27T03:41:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-27T03:41:01,528 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:41:01,563 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-27T03:41:01,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:41:01,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 2a2c4046a8362f1943854d807513f4b6, disabling compactions & flushes 2024-11-27T03:41:01,802 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:01,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:01,802 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. after waiting 0 ms 2024-11-27T03:41:01,803 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:01,803 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:01,803 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a2c4046a8362f1943854d807513f4b6: Waiting for close lock at 1732678861802Disabling compacts and flushes for region at 1732678861802Disabling writes for close at 1732678861803 (+1 ms)Writing region close event to WAL at 1732678861803Closed at 1732678861803 2024-11-27T03:41:01,807 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:41:01,808 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732678861807"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678861807"}]},"ts":"1732678861807"} 2024-11-27T03:41:01,813 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:41:01,814 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:41:01,814 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678861814"}]},"ts":"1732678861814"} 2024-11-27T03:41:01,817 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-27T03:41:01,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=2a2c4046a8362f1943854d807513f4b6, ASSIGN}] 2024-11-27T03:41:01,819 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=2a2c4046a8362f1943854d807513f4b6, ASSIGN 2024-11-27T03:41:01,820 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=2a2c4046a8362f1943854d807513f4b6, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:41:01,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2a2c4046a8362f1943854d807513f4b6, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:41:01,981 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=2a2c4046a8362f1943854d807513f4b6, ASSIGN because future has completed 2024-11-27T03:41:01,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a2c4046a8362f1943854d807513f4b6, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:41:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T03:41:02,140 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:02,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a2c4046a8362f1943854d807513f4b6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:41:02,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:41:02,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,147 INFO [StoreOpener-2a2c4046a8362f1943854d807513f4b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,151 INFO [StoreOpener-2a2c4046a8362f1943854d807513f4b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a2c4046a8362f1943854d807513f4b6 columnFamilyName testFamily 2024-11-27T03:41:02,151 DEBUG [StoreOpener-2a2c4046a8362f1943854d807513f4b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:41:02,156 INFO [StoreOpener-2a2c4046a8362f1943854d807513f4b6-1 {}] regionserver.HStore(327): Store=2a2c4046a8362f1943854d807513f4b6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:41:02,156 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,158 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,159 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,159 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,160 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,162 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,166 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:41:02,167 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 2a2c4046a8362f1943854d807513f4b6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67582891, jitterRate=0.007063552737236023}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:41:02,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:02,168 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 2a2c4046a8362f1943854d807513f4b6: Running coprocessor pre-open hook at 1732678862142Writing region info on filesystem at 1732678862142Initializing all the Stores at 1732678862147 (+5 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678862147Cleaning up temporary data from old regions at 1732678862160 (+13 ms)Running coprocessor post-open hooks at 1732678862167 (+7 ms)Region opened successfully at 1732678862168 (+1 ms) 2024-11-27T03:41:02,169 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., pid=27, masterSystemTime=1732678862134 2024-11-27T03:41:02,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:02,174 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:02,175 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2a2c4046a8362f1943854d807513f4b6, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:41:02,178 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a2c4046a8362f1943854d807513f4b6, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:41:02,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-27T03:41:02,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a2c4046a8362f1943854d807513f4b6, server=66d3634da959,43983,1732678785876 in 199 msec 2024-11-27T03:41:02,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T03:41:02,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=2a2c4046a8362f1943854d807513f4b6, ASSIGN in 368 msec 2024-11-27T03:41:02,189 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:41:02,190 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678862190"}]},"ts":"1732678862190"} 2024-11-27T03:41:02,192 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-27T03:41:02,194 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:41:02,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2260 sec 2024-11-27T03:41:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T03:41:04,003 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-27T03:41:04,003 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-27T03:41:04,003 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:41:04,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-27T03:41:04,011 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:41:04,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-27T03:41:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:41:04,028 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:41:04,028 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-27T03:41:04,033 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T03:41:04,034 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-27T03:41:05,428 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-27T03:41:05,428 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-27T03:41:05,429 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-27T03:41:05,429 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T03:41:07,644 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:41:07,681 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-27T03:41:10,039 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., hostname=66d3634da959,43983,1732678785876, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., hostname=66d3634da959,43983,1732678785876, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=66d3634da959:43983 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-27T03:41:10,040 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., hostname=66d3634da959,43983,1732678785876, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-27T03:41:10,040 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., hostname=66d3634da959,43983,1732678785876, seqNum=2 from cache 2024-11-27T03:41:10,056 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=219 (was 212) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8952 (was 8040) - AvailableMemoryMB LEAK? - 2024-11-27T03:41:10,066 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=8952 2024-11-27T03:41:10,069 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:41:10,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-27T03:41:10,073 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:41:10,073 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:41:10,073 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-27T03:41:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T03:41:10,075 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:41:10,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741851_1027 (size=407) 2024-11-27T03:41:10,489 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb3cbfce443dd71206744401bba3cf59, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:41:10,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741852_1028 (size=58) 2024-11-27T03:41:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-27T03:41:10,905 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:41:10,906 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing eb3cbfce443dd71206744401bba3cf59, disabling compactions & flushes 2024-11-27T03:41:10,906 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:10,906 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:10,906 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. after waiting 0 ms 2024-11-27T03:41:10,906 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:10,906 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:10,907 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb3cbfce443dd71206744401bba3cf59: Waiting for close lock at 1732678870906Disabling compacts and flushes for region at 1732678870906Disabling writes for close at 1732678870906Writing region close event to WAL at 1732678870906Closed at 1732678870906 2024-11-27T03:41:10,910 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:41:10,911 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732678870910"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678870910"}]},"ts":"1732678870910"} 2024-11-27T03:41:10,915 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:41:10,917 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:41:10,917 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678870917"}]},"ts":"1732678870917"} 2024-11-27T03:41:10,921 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-27T03:41:10,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eb3cbfce443dd71206744401bba3cf59, ASSIGN}] 2024-11-27T03:41:10,922 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eb3cbfce443dd71206744401bba3cf59, ASSIGN 2024-11-27T03:41:10,924 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eb3cbfce443dd71206744401bba3cf59, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:41:11,075 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=eb3cbfce443dd71206744401bba3cf59, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:41:11,081 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eb3cbfce443dd71206744401bba3cf59, ASSIGN because future has completed 2024-11-27T03:41:11,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb3cbfce443dd71206744401bba3cf59, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:41:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T03:41:11,245 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:11,246 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => eb3cbfce443dd71206744401bba3cf59, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:41:11,247 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,247 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:41:11,247 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,247 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,251 INFO [StoreOpener-eb3cbfce443dd71206744401bba3cf59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,252 INFO [StoreOpener-eb3cbfce443dd71206744401bba3cf59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb3cbfce443dd71206744401bba3cf59 columnFamilyName testFamily 2024-11-27T03:41:11,252 DEBUG [StoreOpener-eb3cbfce443dd71206744401bba3cf59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:41:11,253 INFO [StoreOpener-eb3cbfce443dd71206744401bba3cf59-1 {}] regionserver.HStore(327): Store=eb3cbfce443dd71206744401bba3cf59/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:41:11,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,254 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,255 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,255 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,255 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,260 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:41:11,261 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened eb3cbfce443dd71206744401bba3cf59; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66649601, jitterRate=-0.006843551993370056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:41:11,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:11,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for eb3cbfce443dd71206744401bba3cf59: Running coprocessor pre-open hook at 1732678871247Writing region info on filesystem at 1732678871247Initializing all the Stores at 1732678871250 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678871250Cleaning up temporary data from old regions at 1732678871255 (+5 ms)Running coprocessor post-open hooks at 1732678871261 (+6 ms)Region opened successfully at 1732678871262 (+1 ms) 2024-11-27T03:41:11,263 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59., pid=30, masterSystemTime=1732678871237 2024-11-27T03:41:11,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:11,267 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:11,267 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=eb3cbfce443dd71206744401bba3cf59, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:41:11,271 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb3cbfce443dd71206744401bba3cf59, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:41:11,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T03:41:11,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure eb3cbfce443dd71206744401bba3cf59, server=66d3634da959,43983,1732678785876 in 190 msec 2024-11-27T03:41:11,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-27T03:41:11,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eb3cbfce443dd71206744401bba3cf59, ASSIGN in 354 msec 2024-11-27T03:41:11,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:41:11,280 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678871280"}]},"ts":"1732678871280"} 2024-11-27T03:41:11,283 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-27T03:41:11,284 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:41:11,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2150 sec 2024-11-27T03:41:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T03:41:13,103 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-27T03:41:13,104 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-27T03:41:13,104 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:41:13,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-27T03:41:13,111 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:41:13,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-27T03:41:13,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:41:13,128 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:41:13,128 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-27T03:41:13,131 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T03:41:13,134 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T03:41:13,135 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T03:41:13,136 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-27T03:41:13,683 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-27T03:41:14,039 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T03:41:15,428 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-27T03:41:15,429 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-27T03:41:16,750 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:41:19,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731572' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T03:41:19,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43983 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:44326 deadline: 1732678885140, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731572' does not exist 2024-11-27T03:41:19,146 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 66d3634da959,43983,1732678785876 for closing scanner id = 6384422693130731572 for eb3cbfce443dd71206744401bba3cf59 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6384422693130731572' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T03:41:19,152 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 219), ProcessCount=11 (was 11), AvailableMemoryMB=8771 (was 8952) 2024-11-27T03:41:19,160 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=8770 2024-11-27T03:41:19,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T03:41:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-27T03:41:19,165 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T03:41:19,165 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:41:19,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-27T03:41:19,166 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T03:41:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T03:41:19,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741853_1029 (size=423) 2024-11-27T03:41:19,579 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9ca111a9300ae87355b91a453abd5c12, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e 2024-11-27T03:41:19,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741854_1030 (size=74) 2024-11-27T03:41:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-27T03:41:19,976 DEBUG [master/66d3634da959:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T03:41:19,989 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:41:19,989 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9ca111a9300ae87355b91a453abd5c12, disabling compactions & flushes 2024-11-27T03:41:19,989 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:19,989 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:19,989 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. after waiting 0 ms 2024-11-27T03:41:19,989 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:19,989 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:19,990 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9ca111a9300ae87355b91a453abd5c12: Waiting for close lock at 1732678879989Disabling compacts and flushes for region at 1732678879989Disabling writes for close at 1732678879989Writing region close event to WAL at 1732678879989Closed at 1732678879989 2024-11-27T03:41:19,991 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T03:41:19,991 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732678879991"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732678879991"}]},"ts":"1732678879991"} 2024-11-27T03:41:19,994 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T03:41:19,995 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T03:41:19,995 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678879995"}]},"ts":"1732678879995"} 2024-11-27T03:41:19,998 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-27T03:41:19,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ca111a9300ae87355b91a453abd5c12, ASSIGN}] 2024-11-27T03:41:20,000 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ca111a9300ae87355b91a453abd5c12, ASSIGN 2024-11-27T03:41:20,002 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ca111a9300ae87355b91a453abd5c12, ASSIGN; state=OFFLINE, location=66d3634da959,43983,1732678785876; forceNewPlan=false, retain=false 2024-11-27T03:41:20,153 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9ca111a9300ae87355b91a453abd5c12, regionState=OPENING, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:41:20,160 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ca111a9300ae87355b91a453abd5c12, ASSIGN because future has completed 2024-11-27T03:41:20,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ca111a9300ae87355b91a453abd5c12, server=66d3634da959,43983,1732678785876}] 2024-11-27T03:41:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T03:41:20,324 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:20,324 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ca111a9300ae87355b91a453abd5c12, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12.', STARTKEY => '', ENDKEY => ''} 2024-11-27T03:41:20,325 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,325 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T03:41:20,325 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,325 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,329 INFO [StoreOpener-9ca111a9300ae87355b91a453abd5c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,333 INFO [StoreOpener-9ca111a9300ae87355b91a453abd5c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ca111a9300ae87355b91a453abd5c12 columnFamilyName testFamily 2024-11-27T03:41:20,333 DEBUG [StoreOpener-9ca111a9300ae87355b91a453abd5c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T03:41:20,334 INFO [StoreOpener-9ca111a9300ae87355b91a453abd5c12-1 {}] regionserver.HStore(327): Store=9ca111a9300ae87355b91a453abd5c12/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T03:41:20,334 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,336 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,337 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,338 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,338 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,342 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,346 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T03:41:20,347 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9ca111a9300ae87355b91a453abd5c12; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65050250, jitterRate=-0.0306757390499115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T03:41:20,347 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:20,348 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9ca111a9300ae87355b91a453abd5c12: Running coprocessor pre-open hook at 1732678880326Writing region info on filesystem at 1732678880326Initializing all the Stores at 1732678880328 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732678880328Cleaning up temporary data from old regions at 1732678880338 (+10 ms)Running coprocessor post-open hooks at 1732678880347 (+9 ms)Region opened successfully at 1732678880348 (+1 ms) 2024-11-27T03:41:20,350 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., pid=33, masterSystemTime=1732678880315 2024-11-27T03:41:20,353 DEBUG [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:20,353 INFO [RS_OPEN_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:20,354 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9ca111a9300ae87355b91a453abd5c12, regionState=OPEN, openSeqNum=2, regionLocation=66d3634da959,43983,1732678785876 2024-11-27T03:41:20,357 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ca111a9300ae87355b91a453abd5c12, server=66d3634da959,43983,1732678785876 because future has completed 2024-11-27T03:41:20,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-27T03:41:20,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ca111a9300ae87355b91a453abd5c12, server=66d3634da959,43983,1732678785876 in 197 msec 2024-11-27T03:41:20,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T03:41:20,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9ca111a9300ae87355b91a453abd5c12, ASSIGN in 361 msec 2024-11-27T03:41:20,363 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T03:41:20,363 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732678880363"}]},"ts":"1732678880363"} 2024-11-27T03:41:20,366 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-27T03:41:20,367 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T03:41:20,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2050 sec 2024-11-27T03:41:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35691 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T03:41:22,196 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-27T03:41:22,196 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-27T03:41:22,197 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:41:22,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-27T03:41:22,205 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T03:41:22,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-27T03:41:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:41:22,221 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T03:41:22,221 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-27T03:41:22,224 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T03:41:22,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T03:41:23,229 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=66d3634da959:43983 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-27T03:41:23,230 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-27T03:41:23,230 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2 from cache 2024-11-27T03:41:23,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-27T03:41:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2] 2024-11-27T03:41:24,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43983 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T03:41:25,229 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=66d3634da959:43983 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=980ms, rpcTimeout=975ms 2024-11-27T03:41:25,229 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=980ms, rpcTimeout=975ms 2024-11-27T03:41:25,229 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., hostname=66d3634da959,43983,1732678785876, seqNum=2 from cache 2024-11-27T03:41:25,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=8764 (was 8770) 2024-11-27T03:41:25,246 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T03:41:25,246 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T03:41:25,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:41:25,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:41:25,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T03:41:25,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T03:41:25,252 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T03:41:25,253 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T03:41:25,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:41:25,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:41:25,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T03:41:25,253 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T03:41:25,253 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=901566946, stopped=false 2024-11-27T03:41:25,253 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=66d3634da959,35691,1732678785028 2024-11-27T03:41:25,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T03:41:25,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T03:41:25,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:41:25,304 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T03:41:25,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:41:25,305 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T03:41:25,305 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T03:41:25,305 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T03:41:25,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:41:25,305 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T03:41:25,305 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '66d3634da959,43983,1732678785876' ***** 2024-11-27T03:41:25,305 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T03:41:25,306 INFO [RS:0;66d3634da959:43983 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T03:41:25,306 INFO [RS:0;66d3634da959:43983 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T03:41:25,306 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T03:41:25,306 INFO [RS:0;66d3634da959:43983 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T03:41:25,306 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for 8f4cc431e1f47cb481fdc44a56ad6342 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for c51a4c2e130a00f12e17466aa3cb9eb0 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for 3fc02201540a0ca11e341f69086bc8a6 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for 2ed84224f9a878accc61e9c054395e36 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for 9ca111a9300ae87355b91a453abd5c12 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for 2a2c4046a8362f1943854d807513f4b6 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for e2ec5a720546e890339825e7278cf91d 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for e28f02d18ffad4f680bcdc3c36dbe311 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:25,307 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8f4cc431e1f47cb481fdc44a56ad6342, disabling compactions & flushes 2024-11-27T03:41:25,307 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:41:25,307 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:41:25,307 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:41:25,308 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(959): stopping server 66d3634da959,43983,1732678785876 2024-11-27T03:41:25,308 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. after waiting 0 ms 2024-11-27T03:41:25,308 INFO [RS:0;66d3634da959:43983 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T03:41:25,308 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:41:25,308 INFO [RS:0;66d3634da959:43983 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;66d3634da959:43983. 2024-11-27T03:41:25,308 DEBUG [RS:0;66d3634da959:43983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T03:41:25,308 DEBUG [RS:0;66d3634da959:43983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:41:25,308 INFO [RS:0;66d3634da959:43983 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T03:41:25,308 INFO [RS:0;66d3634da959:43983 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T03:41:25,308 INFO [RS:0;66d3634da959:43983 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T03:41:25,308 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T03:41:25,309 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-27T03:41:25,309 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T03:41:25,309 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T03:41:25,309 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T03:41:25,309 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T03:41:25,309 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T03:41:25,309 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 8f4cc431e1f47cb481fdc44a56ad6342, 9ca111a9300ae87355b91a453abd5c12, c51a4c2e130a00f12e17466aa3cb9eb0, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:25,311 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-27T03:41:25,311 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8f4cc431e1f47cb481fdc44a56ad6342 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:25,365 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342/.tmp/testFamily/09af0f8318b94060b8cf9ac1991181a5 is 61, key is row-0/testFamily:testQualifier/1732678845718/Put/seqid=0 2024-11-27T03:41:25,369 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/info/32ca114d3c1646c98b46e347f0f26f25 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f./info:regioninfo/1732678820988/Put/seqid=0 2024-11-27T03:41:25,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741855_1031 (size=5049) 2024-11-27T03:41:25,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741856_1032 (size=20564) 2024-11-27T03:41:25,509 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 8f4cc431e1f47cb481fdc44a56ad6342, 9ca111a9300ae87355b91a453abd5c12, c51a4c2e130a00f12e17466aa3cb9eb0, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:25,710 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 8f4cc431e1f47cb481fdc44a56ad6342, 9ca111a9300ae87355b91a453abd5c12, c51a4c2e130a00f12e17466aa3cb9eb0, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:25,751 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-27T03:41:25,777 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342/.tmp/testFamily/09af0f8318b94060b8cf9ac1991181a5 2024-11-27T03:41:25,778 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/info/32ca114d3c1646c98b46e347f0f26f25 2024-11-27T03:41:25,823 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342/.tmp/testFamily/09af0f8318b94060b8cf9ac1991181a5 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342/testFamily/09af0f8318b94060b8cf9ac1991181a5 2024-11-27T03:41:25,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:41:25,832 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342/testFamily/09af0f8318b94060b8cf9ac1991181a5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:25,839 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/ns/ff1650909e3b4843a506f18a7b954f55 is 43, key is default/ns:d/1732678819827/Put/seqid=0 2024-11-27T03:41:25,840 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8f4cc431e1f47cb481fdc44a56ad6342 in 528ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:25,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741857_1033 (size=5153) 2024-11-27T03:41:25,853 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScanner/8f4cc431e1f47cb481fdc44a56ad6342/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:25,855 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:41:25,855 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8f4cc431e1f47cb481fdc44a56ad6342: Waiting for close lock at 1732678885307Running coprocessor pre-close hooks at 1732678885307Disabling compacts and flushes for region at 1732678885307Disabling writes for close at 1732678885308 (+1 ms)Obtaining lock to block concurrent updates at 1732678885311 (+3 ms)Preparing flush snapshotting stores in 8f4cc431e1f47cb481fdc44a56ad6342 at 1732678885311Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678885318 (+7 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. at 1732678885320 (+2 ms)Flushing 8f4cc431e1f47cb481fdc44a56ad6342/testFamily: creating writer at 1732678885321 (+1 ms)Flushing 8f4cc431e1f47cb481fdc44a56ad6342/testFamily: appending metadata at 1732678885360 (+39 ms)Flushing 8f4cc431e1f47cb481fdc44a56ad6342/testFamily: closing flushed file at 1732678885363 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2893e9e2: reopening flushed file at 1732678885821 (+458 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8f4cc431e1f47cb481fdc44a56ad6342 in 528ms, sequenceid=8, compaction requested=false at 1732678885840 (+19 ms)Writing region close event to WAL at 1732678885843 (+3 ms)Running coprocessor post-close hooks at 1732678885854 (+11 ms)Closed at 1732678885855 (+1 ms) 2024-11-27T03:41:25,856 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732678842663.8f4cc431e1f47cb481fdc44a56ad6342. 2024-11-27T03:41:25,856 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c51a4c2e130a00f12e17466aa3cb9eb0, disabling compactions & flushes 2024-11-27T03:41:25,856 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:41:25,856 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:41:25,856 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. after waiting 0 ms 2024-11-27T03:41:25,856 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:41:25,856 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c51a4c2e130a00f12e17466aa3cb9eb0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:25,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-27T03:41:25,864 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T03:41:25,865 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-27T03:41:25,865 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-27T03:41:25,873 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0/.tmp/testFamily/4fa2b433221c4c738835bb79b45612b9 is 61, key is row-0/testFamily:testQualifier/1732678842599/Put/seqid=0 2024-11-27T03:41:25,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741858_1034 (size=5049) 2024-11-27T03:41:25,910 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 9ca111a9300ae87355b91a453abd5c12, c51a4c2e130a00f12e17466aa3cb9eb0, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:26,018 INFO [regionserver/66d3634da959:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T03:41:26,018 INFO [regionserver/66d3634da959:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T03:41:26,110 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 9ca111a9300ae87355b91a453abd5c12, c51a4c2e130a00f12e17466aa3cb9eb0, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:26,252 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/ns/ff1650909e3b4843a506f18a7b954f55 2024-11-27T03:41:26,279 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0/.tmp/testFamily/4fa2b433221c4c738835bb79b45612b9 2024-11-27T03:41:26,282 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/table/44fbb2ec8bb04039b9f33ef7a865494d is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732678821008/Put/seqid=0 2024-11-27T03:41:26,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741859_1035 (size=6699) 2024-11-27T03:41:26,289 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0/.tmp/testFamily/4fa2b433221c4c738835bb79b45612b9 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0/testFamily/4fa2b433221c4c738835bb79b45612b9 2024-11-27T03:41:26,297 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0/testFamily/4fa2b433221c4c738835bb79b45612b9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:26,298 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c51a4c2e130a00f12e17466aa3cb9eb0 in 442ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:26,303 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextException/c51a4c2e130a00f12e17466aa3cb9eb0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:26,304 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:41:26,304 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c51a4c2e130a00f12e17466aa3cb9eb0: Waiting for close lock at 1732678885856Running coprocessor pre-close hooks at 1732678885856Disabling compacts and flushes for region at 1732678885856Disabling writes for close at 1732678885856Obtaining lock to block concurrent updates at 1732678885856Preparing flush snapshotting stores in c51a4c2e130a00f12e17466aa3cb9eb0 at 1732678885856Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678885857 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. at 1732678885857Flushing c51a4c2e130a00f12e17466aa3cb9eb0/testFamily: creating writer at 1732678885857Flushing c51a4c2e130a00f12e17466aa3cb9eb0/testFamily: appending metadata at 1732678885873 (+16 ms)Flushing c51a4c2e130a00f12e17466aa3cb9eb0/testFamily: closing flushed file at 1732678885873Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7da441aa: reopening flushed file at 1732678886288 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c51a4c2e130a00f12e17466aa3cb9eb0 in 442ms, sequenceid=8, compaction requested=false at 1732678886298 (+10 ms)Writing region close event to WAL at 1732678886299 (+1 ms)Running coprocessor post-close hooks at 1732678886304 (+5 ms)Closed at 1732678886304 2024-11-27T03:41:26,304 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732678839544.c51a4c2e130a00f12e17466aa3cb9eb0. 2024-11-27T03:41:26,304 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3fc02201540a0ca11e341f69086bc8a6, disabling compactions & flushes 2024-11-27T03:41:26,305 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:41:26,305 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:41:26,305 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. after waiting 0 ms 2024-11-27T03:41:26,305 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:41:26,305 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3fc02201540a0ca11e341f69086bc8a6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:26,311 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-27T03:41:26,311 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1325): Online Regions={3fc02201540a0ca11e341f69086bc8a6=hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6., 2ed84224f9a878accc61e9c054395e36=testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36., 1588230740=hbase:meta,,1.1588230740, 9ca111a9300ae87355b91a453abd5c12=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., 2a2c4046a8362f1943854d807513f4b6=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., e2ec5a720546e890339825e7278cf91d=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., e28f02d18ffad4f680bcdc3c36dbe311=testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311., eb3cbfce443dd71206744401bba3cf59=hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59., da5e2d6a51e95d62df0d394d31d3005f=testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.} 2024-11-27T03:41:26,311 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 9ca111a9300ae87355b91a453abd5c12, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:26,321 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6/.tmp/testFamily/d0321a7cbf0e4516af782dc53ec39b42 is 61, key is row-0/testFamily:testQualifier/1732678833499/Put/seqid=0 2024-11-27T03:41:26,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741860_1036 (size=5049) 2024-11-27T03:41:26,511 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 9ca111a9300ae87355b91a453abd5c12, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:26,690 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/table/44fbb2ec8bb04039b9f33ef7a865494d 2024-11-27T03:41:26,706 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/info/32ca114d3c1646c98b46e347f0f26f25 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/info/32ca114d3c1646c98b46e347f0f26f25 2024-11-27T03:41:26,712 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 3fc02201540a0ca11e341f69086bc8a6, 9ca111a9300ae87355b91a453abd5c12, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:26,715 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/info/32ca114d3c1646c98b46e347f0f26f25, entries=100, sequenceid=56, filesize=20.1 K 2024-11-27T03:41:26,717 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/ns/ff1650909e3b4843a506f18a7b954f55 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/ns/ff1650909e3b4843a506f18a7b954f55 2024-11-27T03:41:26,726 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/ns/ff1650909e3b4843a506f18a7b954f55, entries=2, sequenceid=56, filesize=5.0 K 2024-11-27T03:41:26,726 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6/.tmp/testFamily/d0321a7cbf0e4516af782dc53ec39b42 2024-11-27T03:41:26,728 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/.tmp/table/44fbb2ec8bb04039b9f33ef7a865494d as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/table/44fbb2ec8bb04039b9f33ef7a865494d 2024-11-27T03:41:26,735 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6/.tmp/testFamily/d0321a7cbf0e4516af782dc53ec39b42 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6/testFamily/d0321a7cbf0e4516af782dc53ec39b42 2024-11-27T03:41:26,736 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/table/44fbb2ec8bb04039b9f33ef7a865494d, entries=20, sequenceid=56, filesize=6.5 K 2024-11-27T03:41:26,738 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1429ms, sequenceid=56, compaction requested=false 2024-11-27T03:41:26,744 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6/testFamily/d0321a7cbf0e4516af782dc53ec39b42, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:26,744 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-27T03:41:26,745 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T03:41:26,745 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3fc02201540a0ca11e341f69086bc8a6 in 440ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:26,745 INFO [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T03:41:26,745 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732678885309Running coprocessor pre-close hooks at 1732678885309Disabling compacts and flushes for region at 1732678885309Disabling writes for close at 1732678885309Obtaining lock to block concurrent updates at 1732678885311 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732678885311Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732678885318 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732678885320 (+2 ms)Flushing 1588230740/info: creating writer at 1732678885321 (+1 ms)Flushing 1588230740/info: appending metadata at 1732678885365 (+44 ms)Flushing 1588230740/info: closing flushed file at 1732678885365Flushing 1588230740/ns: creating writer at 1732678885821 (+456 ms)Flushing 1588230740/ns: appending metadata at 1732678885838 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732678885838Flushing 1588230740/table: creating writer at 1732678886266 (+428 ms)Flushing 1588230740/table: appending metadata at 1732678886282 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732678886282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ee9191: reopening flushed file at 1732678886705 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c04c63c: reopening flushed file at 1732678886716 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76f2a717: reopening flushed file at 1732678886726 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1429ms, sequenceid=56, compaction requested=false at 1732678886738 (+12 ms)Writing region close event to WAL at 1732678886740 (+2 ms)Running coprocessor post-close hooks at 1732678886745 (+5 ms)Closed at 1732678886745 2024-11-27T03:41:26,745 DEBUG [RS_CLOSE_META-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T03:41:26,749 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNextAsync/3fc02201540a0ca11e341f69086bc8a6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:26,750 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:41:26,750 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3fc02201540a0ca11e341f69086bc8a6: Waiting for close lock at 1732678886304Running coprocessor pre-close hooks at 1732678886304Disabling compacts and flushes for region at 1732678886304Disabling writes for close at 1732678886305 (+1 ms)Obtaining lock to block concurrent updates at 1732678886305Preparing flush snapshotting stores in 3fc02201540a0ca11e341f69086bc8a6 at 1732678886305Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678886305Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. at 1732678886306 (+1 ms)Flushing 3fc02201540a0ca11e341f69086bc8a6/testFamily: creating writer at 1732678886306Flushing 3fc02201540a0ca11e341f69086bc8a6/testFamily: appending metadata at 1732678886321 (+15 ms)Flushing 3fc02201540a0ca11e341f69086bc8a6/testFamily: closing flushed file at 1732678886321Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bbead63: reopening flushed file at 1732678886734 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3fc02201540a0ca11e341f69086bc8a6 in 440ms, sequenceid=8, compaction requested=false at 1732678886745 (+11 ms)Writing region close event to WAL at 1732678886745Running coprocessor post-close hooks at 1732678886750 (+5 ms)Closed at 1732678886750 2024-11-27T03:41:26,750 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732678830452.3fc02201540a0ca11e341f69086bc8a6. 2024-11-27T03:41:26,751 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2ed84224f9a878accc61e9c054395e36, disabling compactions & flushes 2024-11-27T03:41:26,751 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:41:26,751 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:41:26,751 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. after waiting 0 ms 2024-11-27T03:41:26,751 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:41:26,751 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2ed84224f9a878accc61e9c054395e36 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:26,767 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36/.tmp/testFamily/60537da90e164aa69e512ed70d3ab443 is 61, key is row-0/testFamily:testQualifier/1732678851820/Put/seqid=0 2024-11-27T03:41:26,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741861_1037 (size=5049) 2024-11-27T03:41:26,912 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 9ca111a9300ae87355b91a453abd5c12, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:27,113 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 2a2c4046a8362f1943854d807513f4b6, 2ed84224f9a878accc61e9c054395e36, 9ca111a9300ae87355b91a453abd5c12, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:27,173 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36/.tmp/testFamily/60537da90e164aa69e512ed70d3ab443 2024-11-27T03:41:27,187 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36/.tmp/testFamily/60537da90e164aa69e512ed70d3ab443 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36/testFamily/60537da90e164aa69e512ed70d3ab443 2024-11-27T03:41:27,194 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36/testFamily/60537da90e164aa69e512ed70d3ab443, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:27,195 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ed84224f9a878accc61e9c054395e36 in 444ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:27,200 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNextAsync/2ed84224f9a878accc61e9c054395e36/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:27,201 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:41:27,201 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2ed84224f9a878accc61e9c054395e36: Waiting for close lock at 1732678886751Running coprocessor pre-close hooks at 1732678886751Disabling compacts and flushes for region at 1732678886751Disabling writes for close at 1732678886751Obtaining lock to block concurrent updates at 1732678886751Preparing flush snapshotting stores in 2ed84224f9a878accc61e9c054395e36 at 1732678886751Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678886751Flushing stores of testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. at 1732678886752 (+1 ms)Flushing 2ed84224f9a878accc61e9c054395e36/testFamily: creating writer at 1732678886752Flushing 2ed84224f9a878accc61e9c054395e36/testFamily: appending metadata at 1732678886767 (+15 ms)Flushing 2ed84224f9a878accc61e9c054395e36/testFamily: closing flushed file at 1732678886767Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14b811b9: reopening flushed file at 1732678887185 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ed84224f9a878accc61e9c054395e36 in 444ms, sequenceid=8, compaction requested=false at 1732678887195 (+10 ms)Writing region close event to WAL at 1732678887196 (+1 ms)Running coprocessor post-close hooks at 1732678887201 (+5 ms)Closed at 1732678887201 2024-11-27T03:41:27,201 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732678848764.2ed84224f9a878accc61e9c054395e36. 2024-11-27T03:41:27,202 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9ca111a9300ae87355b91a453abd5c12, disabling compactions & flushes 2024-11-27T03:41:27,202 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:27,202 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:27,202 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. after waiting 0 ms 2024-11-27T03:41:27,202 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:27,202 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9ca111a9300ae87355b91a453abd5c12 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:27,218 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12/.tmp/testFamily/dbaa48f85108497ea2b13d5e0e9e00b9 is 61, key is row-0/testFamily:testQualifier/1732678882211/Put/seqid=0 2024-11-27T03:41:27,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741862_1038 (size=5049) 2024-11-27T03:41:27,313 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-27T03:41:27,313 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1325): Online Regions={9ca111a9300ae87355b91a453abd5c12=testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., 2a2c4046a8362f1943854d807513f4b6=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., e2ec5a720546e890339825e7278cf91d=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., e28f02d18ffad4f680bcdc3c36dbe311=testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311., eb3cbfce443dd71206744401bba3cf59=hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59., da5e2d6a51e95d62df0d394d31d3005f=testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.} 2024-11-27T03:41:27,313 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 2a2c4046a8362f1943854d807513f4b6, 9ca111a9300ae87355b91a453abd5c12, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:27,514 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 2a2c4046a8362f1943854d807513f4b6, 9ca111a9300ae87355b91a453abd5c12, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:27,625 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12/.tmp/testFamily/dbaa48f85108497ea2b13d5e0e9e00b9 2024-11-27T03:41:27,642 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12/.tmp/testFamily/dbaa48f85108497ea2b13d5e0e9e00b9 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12/testFamily/dbaa48f85108497ea2b13d5e0e9e00b9 2024-11-27T03:41:27,652 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12/testFamily/dbaa48f85108497ea2b13d5e0e9e00b9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:27,653 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9ca111a9300ae87355b91a453abd5c12 in 451ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:27,658 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9ca111a9300ae87355b91a453abd5c12/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:27,658 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:27,659 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9ca111a9300ae87355b91a453abd5c12: Waiting for close lock at 1732678887202Running coprocessor pre-close hooks at 1732678887202Disabling compacts and flushes for region at 1732678887202Disabling writes for close at 1732678887202Obtaining lock to block concurrent updates at 1732678887202Preparing flush snapshotting stores in 9ca111a9300ae87355b91a453abd5c12 at 1732678887202Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678887202Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. at 1732678887203 (+1 ms)Flushing 9ca111a9300ae87355b91a453abd5c12/testFamily: creating writer at 1732678887203Flushing 9ca111a9300ae87355b91a453abd5c12/testFamily: appending metadata at 1732678887218 (+15 ms)Flushing 9ca111a9300ae87355b91a453abd5c12/testFamily: closing flushed file at 1732678887218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b7b460b: reopening flushed file at 1732678887640 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9ca111a9300ae87355b91a453abd5c12 in 451ms, sequenceid=8, compaction requested=false at 1732678887653 (+13 ms)Writing region close event to WAL at 1732678887654 (+1 ms)Running coprocessor post-close hooks at 1732678887658 (+4 ms)Closed at 1732678887658 2024-11-27T03:41:27,659 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732678879162.9ca111a9300ae87355b91a453abd5c12. 2024-11-27T03:41:27,659 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2a2c4046a8362f1943854d807513f4b6, disabling compactions & flushes 2024-11-27T03:41:27,659 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:27,659 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:27,659 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. after waiting 0 ms 2024-11-27T03:41:27,659 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:27,659 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2a2c4046a8362f1943854d807513f4b6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:27,680 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6/.tmp/testFamily/c6acabaf72894fc38d5e33c029835c5c is 61, key is row-0/testFamily:testQualifier/1732678864016/Put/seqid=0 2024-11-27T03:41:27,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741863_1039 (size=5049) 2024-11-27T03:41:27,714 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 2a2c4046a8362f1943854d807513f4b6, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:27,914 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 2a2c4046a8362f1943854d807513f4b6, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:28,088 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6/.tmp/testFamily/c6acabaf72894fc38d5e33c029835c5c 2024-11-27T03:41:28,102 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6/.tmp/testFamily/c6acabaf72894fc38d5e33c029835c5c as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6/testFamily/c6acabaf72894fc38d5e33c029835c5c 2024-11-27T03:41:28,110 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6/testFamily/c6acabaf72894fc38d5e33c029835c5c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:28,111 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2a2c4046a8362f1943854d807513f4b6 in 452ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:28,114 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on 2a2c4046a8362f1943854d807513f4b6, da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:28,116 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/2a2c4046a8362f1943854d807513f4b6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:28,117 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:28,117 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2a2c4046a8362f1943854d807513f4b6: Waiting for close lock at 1732678887659Running coprocessor pre-close hooks at 1732678887659Disabling compacts and flushes for region at 1732678887659Disabling writes for close at 1732678887659Obtaining lock to block concurrent updates at 1732678887659Preparing flush snapshotting stores in 2a2c4046a8362f1943854d807513f4b6 at 1732678887659Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678887659Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. at 1732678887660 (+1 ms)Flushing 2a2c4046a8362f1943854d807513f4b6/testFamily: creating writer at 1732678887660Flushing 2a2c4046a8362f1943854d807513f4b6/testFamily: appending metadata at 1732678887680 (+20 ms)Flushing 2a2c4046a8362f1943854d807513f4b6/testFamily: closing flushed file at 1732678887680Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a3cd434: reopening flushed file at 1732678888101 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2a2c4046a8362f1943854d807513f4b6 in 452ms, sequenceid=8, compaction requested=false at 1732678888111 (+10 ms)Writing region close event to WAL at 1732678888112 (+1 ms)Running coprocessor post-close hooks at 1732678888116 (+4 ms)Closed at 1732678888116 2024-11-27T03:41:28,117 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732678860968.2a2c4046a8362f1943854d807513f4b6. 2024-11-27T03:41:28,117 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e2ec5a720546e890339825e7278cf91d, disabling compactions & flushes 2024-11-27T03:41:28,117 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:41:28,117 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:41:28,117 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. after waiting 0 ms 2024-11-27T03:41:28,117 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:41:28,117 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e2ec5a720546e890339825e7278cf91d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:28,134 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d/.tmp/testFamily/0dc0b8deffe44bc8a5b3460967261adf is 61, key is row-0/testFamily:testQualifier/1732678824397/Put/seqid=0 2024-11-27T03:41:28,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741864_1040 (size=5049) 2024-11-27T03:41:28,315 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-27T03:41:28,315 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1325): Online Regions={e2ec5a720546e890339825e7278cf91d=hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., e28f02d18ffad4f680bcdc3c36dbe311=testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311., eb3cbfce443dd71206744401bba3cf59=hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59., da5e2d6a51e95d62df0d394d31d3005f=testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.} 2024-11-27T03:41:28,315 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:28,516 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, e2ec5a720546e890339825e7278cf91d, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:28,541 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d/.tmp/testFamily/0dc0b8deffe44bc8a5b3460967261adf 2024-11-27T03:41:28,556 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d/.tmp/testFamily/0dc0b8deffe44bc8a5b3460967261adf as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d/testFamily/0dc0b8deffe44bc8a5b3460967261adf 2024-11-27T03:41:28,566 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d/testFamily/0dc0b8deffe44bc8a5b3460967261adf, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:28,567 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e2ec5a720546e890339825e7278cf91d in 450ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:28,573 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnOpenScanner/e2ec5a720546e890339825e7278cf91d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:28,574 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:41:28,574 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e2ec5a720546e890339825e7278cf91d: Waiting for close lock at 1732678888117Running coprocessor pre-close hooks at 1732678888117Disabling compacts and flushes for region at 1732678888117Disabling writes for close at 1732678888117Obtaining lock to block concurrent updates at 1732678888117Preparing flush snapshotting stores in e2ec5a720546e890339825e7278cf91d at 1732678888117Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678888118 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. at 1732678888118Flushing e2ec5a720546e890339825e7278cf91d/testFamily: creating writer at 1732678888118Flushing e2ec5a720546e890339825e7278cf91d/testFamily: appending metadata at 1732678888134 (+16 ms)Flushing e2ec5a720546e890339825e7278cf91d/testFamily: closing flushed file at 1732678888134Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb3c3b3: reopening flushed file at 1732678888555 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e2ec5a720546e890339825e7278cf91d in 450ms, sequenceid=8, compaction requested=false at 1732678888567 (+12 ms)Writing region close event to WAL at 1732678888568 (+1 ms)Running coprocessor post-close hooks at 1732678888574 (+6 ms)Closed at 1732678888574 2024-11-27T03:41:28,575 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732678821330.e2ec5a720546e890339825e7278cf91d. 2024-11-27T03:41:28,575 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e28f02d18ffad4f680bcdc3c36dbe311, disabling compactions & flushes 2024-11-27T03:41:28,575 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:41:28,575 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:41:28,575 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. after waiting 0 ms 2024-11-27T03:41:28,575 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:41:28,575 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e28f02d18ffad4f680bcdc3c36dbe311 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:28,591 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311/.tmp/testFamily/87af796330d946f593c36a43b26582f4 is 61, key is row-0/testFamily:testQualifier/1732678857920/Put/seqid=0 2024-11-27T03:41:28,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741865_1041 (size=5049) 2024-11-27T03:41:28,716 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:28,917 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f, e28f02d18ffad4f680bcdc3c36dbe311, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:28,998 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311/.tmp/testFamily/87af796330d946f593c36a43b26582f4 2024-11-27T03:41:29,012 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311/.tmp/testFamily/87af796330d946f593c36a43b26582f4 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311/testFamily/87af796330d946f593c36a43b26582f4 2024-11-27T03:41:29,020 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311/testFamily/87af796330d946f593c36a43b26582f4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:29,021 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e28f02d18ffad4f680bcdc3c36dbe311 in 446ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:29,026 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testNormalScanTimeoutOnNext/e28f02d18ffad4f680bcdc3c36dbe311/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:29,027 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:41:29,027 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e28f02d18ffad4f680bcdc3c36dbe311: Waiting for close lock at 1732678888575Running coprocessor pre-close hooks at 1732678888575Disabling compacts and flushes for region at 1732678888575Disabling writes for close at 1732678888575Obtaining lock to block concurrent updates at 1732678888575Preparing flush snapshotting stores in e28f02d18ffad4f680bcdc3c36dbe311 at 1732678888575Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678888576 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. at 1732678888576Flushing e28f02d18ffad4f680bcdc3c36dbe311/testFamily: creating writer at 1732678888577 (+1 ms)Flushing e28f02d18ffad4f680bcdc3c36dbe311/testFamily: appending metadata at 1732678888591 (+14 ms)Flushing e28f02d18ffad4f680bcdc3c36dbe311/testFamily: closing flushed file at 1732678888591Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73f4033b: reopening flushed file at 1732678889011 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e28f02d18ffad4f680bcdc3c36dbe311 in 446ms, sequenceid=8, compaction requested=false at 1732678889021 (+10 ms)Writing region close event to WAL at 1732678889022 (+1 ms)Running coprocessor post-close hooks at 1732678889027 (+5 ms)Closed at 1732678889027 2024-11-27T03:41:29,027 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732678854859.e28f02d18ffad4f680bcdc3c36dbe311. 2024-11-27T03:41:29,027 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb3cbfce443dd71206744401bba3cf59, disabling compactions & flushes 2024-11-27T03:41:29,028 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:29,028 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:29,028 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. after waiting 0 ms 2024-11-27T03:41:29,028 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:29,028 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eb3cbfce443dd71206744401bba3cf59 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:29,048 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59/.tmp/testFamily/5ab6e7a56b5f439e96c582c3f4eaf0a0 is 61, key is row-0/testFamily:testQualifier/1732678873116/Put/seqid=0 2024-11-27T03:41:29,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741866_1042 (size=5049) 2024-11-27T03:41:29,117 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:29,317 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T03:41:29,318 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1325): Online Regions={eb3cbfce443dd71206744401bba3cf59=hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59., da5e2d6a51e95d62df0d394d31d3005f=testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f.} 2024-11-27T03:41:29,318 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f, eb3cbfce443dd71206744401bba3cf59 2024-11-27T03:41:29,454 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59/.tmp/testFamily/5ab6e7a56b5f439e96c582c3f4eaf0a0 2024-11-27T03:41:29,469 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59/.tmp/testFamily/5ab6e7a56b5f439e96c582c3f4eaf0a0 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59/testFamily/5ab6e7a56b5f439e96c582c3f4eaf0a0 2024-11-27T03:41:29,477 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59/testFamily/5ab6e7a56b5f439e96c582c3f4eaf0a0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:29,478 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb3cbfce443dd71206744401bba3cf59 in 450ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:29,484 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/hbase/testMetaScanTimeoutOnNext/eb3cbfce443dd71206744401bba3cf59/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:29,484 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:29,484 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb3cbfce443dd71206744401bba3cf59: Waiting for close lock at 1732678889027Running coprocessor pre-close hooks at 1732678889027Disabling compacts and flushes for region at 1732678889027Disabling writes for close at 1732678889028 (+1 ms)Obtaining lock to block concurrent updates at 1732678889028Preparing flush snapshotting stores in eb3cbfce443dd71206744401bba3cf59 at 1732678889028Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678889028Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. at 1732678889029 (+1 ms)Flushing eb3cbfce443dd71206744401bba3cf59/testFamily: creating writer at 1732678889029Flushing eb3cbfce443dd71206744401bba3cf59/testFamily: appending metadata at 1732678889047 (+18 ms)Flushing eb3cbfce443dd71206744401bba3cf59/testFamily: closing flushed file at 1732678889047Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@755be883: reopening flushed file at 1732678889468 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb3cbfce443dd71206744401bba3cf59 in 450ms, sequenceid=8, compaction requested=false at 1732678889478 (+10 ms)Writing region close event to WAL at 1732678889479 (+1 ms)Running coprocessor post-close hooks at 1732678889484 (+5 ms)Closed at 1732678889484 2024-11-27T03:41:29,485 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732678870068.eb3cbfce443dd71206744401bba3cf59. 2024-11-27T03:41:29,485 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing da5e2d6a51e95d62df0d394d31d3005f, disabling compactions & flushes 2024-11-27T03:41:29,485 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:41:29,485 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:41:29,485 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. after waiting 0 ms 2024-11-27T03:41:29,485 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:41:29,485 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing da5e2d6a51e95d62df0d394d31d3005f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T03:41:29,502 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f/.tmp/testFamily/9b3b93e62b22486db27cda341b46252e is 61, key is row-0/testFamily:testQualifier/1732678821213/Put/seqid=0 2024-11-27T03:41:29,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741867_1043 (size=5049) 2024-11-27T03:41:29,518 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:41:29,719 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:41:29,909 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f/.tmp/testFamily/9b3b93e62b22486db27cda341b46252e 2024-11-27T03:41:29,919 DEBUG [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1351): Waiting on da5e2d6a51e95d62df0d394d31d3005f 2024-11-27T03:41:29,923 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f/.tmp/testFamily/9b3b93e62b22486db27cda341b46252e as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f/testFamily/9b3b93e62b22486db27cda341b46252e 2024-11-27T03:41:29,929 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f/testFamily/9b3b93e62b22486db27cda341b46252e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T03:41:29,930 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for da5e2d6a51e95d62df0d394d31d3005f in 445ms, sequenceid=8, compaction requested=false 2024-11-27T03:41:29,935 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/da5e2d6a51e95d62df0d394d31d3005f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T03:41:29,936 INFO [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:41:29,936 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for da5e2d6a51e95d62df0d394d31d3005f: Waiting for close lock at 1732678889485Running coprocessor pre-close hooks at 1732678889485Disabling compacts and flushes for region at 1732678889485Disabling writes for close at 1732678889485Obtaining lock to block concurrent updates at 1732678889485Preparing flush snapshotting stores in da5e2d6a51e95d62df0d394d31d3005f at 1732678889485Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732678889485Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. at 1732678889486 (+1 ms)Flushing da5e2d6a51e95d62df0d394d31d3005f/testFamily: creating writer at 1732678889486Flushing da5e2d6a51e95d62df0d394d31d3005f/testFamily: appending metadata at 1732678889502 (+16 ms)Flushing da5e2d6a51e95d62df0d394d31d3005f/testFamily: closing flushed file at 1732678889502Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fa92be8: reopening flushed file at 1732678889922 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for da5e2d6a51e95d62df0d394d31d3005f in 445ms, sequenceid=8, compaction requested=false at 1732678889930 (+8 ms)Writing region close event to WAL at 1732678889931 (+1 ms)Running coprocessor post-close hooks at 1732678889936 (+5 ms)Closed at 1732678889936 2024-11-27T03:41:29,936 DEBUG [RS_CLOSE_REGION-regionserver/66d3634da959:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732678820143.da5e2d6a51e95d62df0d394d31d3005f. 2024-11-27T03:41:30,119 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(976): stopping server 66d3634da959,43983,1732678785876; all regions closed. 2024-11-27T03:41:30,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741834_1010 (size=19479) 2024-11-27T03:41:30,134 DEBUG [RS:0;66d3634da959:43983 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/oldWALs 2024-11-27T03:41:30,134 INFO [RS:0;66d3634da959:43983 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 66d3634da959%2C43983%2C1732678785876.meta:.meta(num 1732678819496) 2024-11-27T03:41:30,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741833_1009 (size=20031) 2024-11-27T03:41:30,176 DEBUG [RS:0;66d3634da959:43983 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/oldWALs 2024-11-27T03:41:30,176 INFO [RS:0;66d3634da959:43983 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 66d3634da959%2C43983%2C1732678785876:(num 1732678819079) 2024-11-27T03:41:30,177 DEBUG [RS:0;66d3634da959:43983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T03:41:30,177 INFO [RS:0;66d3634da959:43983 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T03:41:30,177 INFO [RS:0;66d3634da959:43983 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T03:41:30,177 INFO [RS:0;66d3634da959:43983 {}] hbase.ChoreService(370): Chore service for: regionserver/66d3634da959:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T03:41:30,177 INFO [RS:0;66d3634da959:43983 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T03:41:30,177 INFO [regionserver/66d3634da959:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T03:41:30,179 INFO [RS:0;66d3634da959:43983 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43983 2024-11-27T03:41:30,217 INFO [regionserver/66d3634da959:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T03:41:30,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/66d3634da959,43983,1732678785876 2024-11-27T03:41:30,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T03:41:30,226 INFO [RS:0;66d3634da959:43983 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T03:41:30,251 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [66d3634da959,43983,1732678785876] 2024-11-27T03:41:30,262 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/66d3634da959,43983,1732678785876 already deleted, retry=false 2024-11-27T03:41:30,262 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 66d3634da959,43983,1732678785876 expired; onlineServers=0 2024-11-27T03:41:30,263 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '66d3634da959,35691,1732678785028' ***** 2024-11-27T03:41:30,263 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T03:41:30,264 INFO [M:0;66d3634da959:35691 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T03:41:30,264 INFO [M:0;66d3634da959:35691 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T03:41:30,264 DEBUG [M:0;66d3634da959:35691 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T03:41:30,265 DEBUG [M:0;66d3634da959:35691 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T03:41:30,265 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T03:41:30,265 DEBUG [master/66d3634da959:0:becomeActiveMaster-HFileCleaner.large.0-1732678788873 {}] cleaner.HFileCleaner(306): Exit Thread[master/66d3634da959:0:becomeActiveMaster-HFileCleaner.large.0-1732678788873,5,FailOnTimeoutGroup] 2024-11-27T03:41:30,265 DEBUG [master/66d3634da959:0:becomeActiveMaster-HFileCleaner.small.0-1732678788874 {}] cleaner.HFileCleaner(306): Exit Thread[master/66d3634da959:0:becomeActiveMaster-HFileCleaner.small.0-1732678788874,5,FailOnTimeoutGroup] 2024-11-27T03:41:30,266 INFO [M:0;66d3634da959:35691 {}] hbase.ChoreService(370): Chore service for: master/66d3634da959:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T03:41:30,266 INFO [M:0;66d3634da959:35691 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T03:41:30,267 DEBUG [M:0;66d3634da959:35691 {}] master.HMaster(1795): Stopping service threads 2024-11-27T03:41:30,267 INFO [M:0;66d3634da959:35691 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T03:41:30,268 INFO [M:0;66d3634da959:35691 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T03:41:30,269 INFO [M:0;66d3634da959:35691 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T03:41:30,269 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T03:41:30,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T03:41:30,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T03:41:30,272 DEBUG [M:0;66d3634da959:35691 {}] zookeeper.ZKUtil(347): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T03:41:30,272 WARN [M:0;66d3634da959:35691 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T03:41:30,274 INFO [M:0;66d3634da959:35691 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/.lastflushedseqids 2024-11-27T03:41:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741868_1044 (size=649) 2024-11-27T03:41:30,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T03:41:30,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43983-0x1017a6010240001, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T03:41:30,352 INFO [RS:0;66d3634da959:43983 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T03:41:30,352 INFO [RS:0;66d3634da959:43983 {}] regionserver.HRegionServer(1031): Exiting; stopping=66d3634da959,43983,1732678785876; zookeeper connection closed. 2024-11-27T03:41:30,352 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7504cb15 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7504cb15 2024-11-27T03:41:30,353 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T03:41:30,687 INFO [M:0;66d3634da959:35691 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T03:41:30,687 INFO [M:0;66d3634da959:35691 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T03:41:30,687 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T03:41:30,688 INFO [M:0;66d3634da959:35691 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:41:30,688 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:41:30,688 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T03:41:30,688 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:41:30,689 INFO [M:0;66d3634da959:35691 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.08 KB heapSize=190.99 KB 2024-11-27T03:41:30,709 DEBUG [M:0;66d3634da959:35691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6f1cef8bb7c4e8cb423c1bae8f65efc is 82, key is hbase:meta,,1/info:regioninfo/1732678819594/Put/seqid=0 2024-11-27T03:41:30,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741869_1045 (size=5672) 2024-11-27T03:41:31,115 INFO [M:0;66d3634da959:35691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6f1cef8bb7c4e8cb423c1bae8f65efc 2024-11-27T03:41:31,148 DEBUG [M:0;66d3634da959:35691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2caf608c7b5d4bbbb2fc2196bb86279d is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732678821016/Put/seqid=0 2024-11-27T03:41:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741870_1046 (size=14077) 2024-11-27T03:41:31,360 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T03:41:31,555 INFO [M:0;66d3634da959:35691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.53 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2caf608c7b5d4bbbb2fc2196bb86279d 2024-11-27T03:41:31,568 INFO [M:0;66d3634da959:35691 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2caf608c7b5d4bbbb2fc2196bb86279d 2024-11-27T03:41:31,585 DEBUG [M:0;66d3634da959:35691 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c82dc67bbcc649c897c1c23b57b53b06 is 69, key is 66d3634da959,43983,1732678785876/rs:state/1732678788927/Put/seqid=0 2024-11-27T03:41:31,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741871_1047 (size=5156) 2024-11-27T03:41:31,992 INFO [M:0;66d3634da959:35691 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c82dc67bbcc649c897c1c23b57b53b06 2024-11-27T03:41:32,004 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6f1cef8bb7c4e8cb423c1bae8f65efc as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f6f1cef8bb7c4e8cb423c1bae8f65efc 2024-11-27T03:41:32,010 INFO [M:0;66d3634da959:35691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f6f1cef8bb7c4e8cb423c1bae8f65efc, entries=8, sequenceid=317, filesize=5.5 K 2024-11-27T03:41:32,012 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2caf608c7b5d4bbbb2fc2196bb86279d as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2caf608c7b5d4bbbb2fc2196bb86279d 2024-11-27T03:41:32,018 INFO [M:0;66d3634da959:35691 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2caf608c7b5d4bbbb2fc2196bb86279d 2024-11-27T03:41:32,018 INFO [M:0;66d3634da959:35691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2caf608c7b5d4bbbb2fc2196bb86279d, entries=33, sequenceid=317, filesize=13.7 K 2024-11-27T03:41:32,019 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c82dc67bbcc649c897c1c23b57b53b06 as hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c82dc67bbcc649c897c1c23b57b53b06 2024-11-27T03:41:32,026 INFO [M:0;66d3634da959:35691 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35725/user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c82dc67bbcc649c897c1c23b57b53b06, entries=1, sequenceid=317, filesize=5.0 K 2024-11-27T03:41:32,028 INFO [M:0;66d3634da959:35691 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.08 KB/166999, heapSize ~190.70 KB/195272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1339ms, sequenceid=317, compaction requested=false 2024-11-27T03:41:32,029 INFO [M:0;66d3634da959:35691 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T03:41:32,029 DEBUG [M:0;66d3634da959:35691 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732678890687Disabling compacts and flushes for region at 1732678890687Disabling writes for close at 1732678890688 (+1 ms)Obtaining lock to block concurrent updates at 1732678890689 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732678890689Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166999, getHeapSize=195512, getOffHeapSize=0, getCellsCount=364 at 1732678890689Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732678890691 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732678890691Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732678890708 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732678890708Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732678891130 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732678891147 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732678891147Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732678891569 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732678891584 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732678891584Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@393f6f59: reopening flushed file at 1732678892002 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d54ae65: reopening flushed file at 1732678892010 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aa9cfc7: reopening flushed file at 1732678892018 (+8 ms)Finished flush of dataSize ~163.08 KB/166999, heapSize ~190.70 KB/195272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1339ms, sequenceid=317, compaction requested=false at 1732678892028 (+10 ms)Writing region close event to WAL at 1732678892029 (+1 ms)Closed at 1732678892029 2024-11-27T03:41:32,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34855 is added to blk_1073741830_1006 (size=188211) 2024-11-27T03:41:32,034 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/3e0be7f6-257b-ae20-81a9-a45d5933ec8e/MasterData/WALs/66d3634da959,35691,1732678785028/66d3634da959%2C35691%2C1732678785028.1732678788086 not finished, retry = 0 2024-11-27T03:41:32,137 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T03:41:32,137 INFO [M:0;66d3634da959:35691 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T03:41:32,137 INFO [M:0;66d3634da959:35691 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35691 2024-11-27T03:41:32,138 INFO [M:0;66d3634da959:35691 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T03:41:32,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T03:41:32,310 INFO [M:0;66d3634da959:35691 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T03:41:32,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35691-0x1017a6010240000, quorum=127.0.0.1:63169, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T03:41:32,321 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f9a1a82{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T03:41:32,323 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@210254cd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T03:41:32,324 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T03:41:32,324 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23c598c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T03:41:32,324 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7aa82d16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/hadoop.log.dir/,STOPPED} 2024-11-27T03:41:32,326 WARN [BP-1930725240-172.17.0.3-1732678780861 heartbeating to localhost/127.0.0.1:35725 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T03:41:32,326 WARN [BP-1930725240-172.17.0.3-1732678780861 heartbeating to localhost/127.0.0.1:35725 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1930725240-172.17.0.3-1732678780861 (Datanode Uuid 9c1f62c8-f8b1-4269-93ae-d9bab8226688) service to localhost/127.0.0.1:35725 2024-11-27T03:41:32,326 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T03:41:32,326 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T03:41:32,327 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10/data/data1/current/BP-1930725240-172.17.0.3-1732678780861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T03:41:32,328 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/cluster_203819cd-918b-8f3c-8baa-97f052fc4f10/data/data2/current/BP-1930725240-172.17.0.3-1732678780861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T03:41:32,328 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T03:41:32,336 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4df87c21{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T03:41:32,337 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T03:41:32,337 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T03:41:32,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T03:41:32,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87bd1f81-2de4-3bde-d762-e3d03b8e435d/hadoop.log.dir/,STOPPED} 2024-11-27T03:41:32,345 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T03:41:32,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down