2024-12-08 08:27:22,658 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-08 08:27:22,677 main DEBUG Took 0.017553 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 08:27:22,678 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 08:27:22,678 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 08:27:22,679 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 08:27:22,680 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,687 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 08:27:22,700 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,702 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,703 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,704 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,705 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,706 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,707 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,708 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,709 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,709 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,711 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,711 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,713 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,714 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 08:27:22,715 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,715 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 08:27:22,717 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 08:27:22,719 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 08:27:22,721 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 08:27:22,722 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 08:27:22,723 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 08:27:22,724 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 08:27:22,734 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 08:27:22,736 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 08:27:22,738 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 08:27:22,738 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 08:27:22,738 main DEBUG createAppenders(={Console}) 2024-12-08 08:27:22,739 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-08 08:27:22,739 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-08 08:27:22,740 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-08 08:27:22,740 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 08:27:22,740 main DEBUG OutputStream closed 2024-12-08 08:27:22,740 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 08:27:22,741 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 08:27:22,741 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-08 08:27:22,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 08:27:22,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 08:27:22,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 08:27:22,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 08:27:22,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 08:27:22,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 08:27:22,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 08:27:22,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 08:27:22,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 08:27:22,838 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 08:27:22,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 08:27:22,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 08:27:22,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 08:27:22,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 08:27:22,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 08:27:22,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 08:27:22,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 08:27:22,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 08:27:22,844 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 08:27:22,845 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-08 08:27:22,845 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 08:27:22,846 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-08T08:27:22,862 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-08 08:27:22,865 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 08:27:22,866 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T08:27:23,140 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3 2024-12-08T08:27:23,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T08:27:23,177 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442, deleteOnExit=true 2024-12-08T08:27:23,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T08:27:23,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/test.cache.data in system properties and HBase conf 2024-12-08T08:27:23,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T08:27:23,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/hadoop.log.dir in system properties and HBase conf 2024-12-08T08:27:23,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T08:27:23,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T08:27:23,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T08:27:23,288 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T08:27:23,398 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T08:27:23,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T08:27:23,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T08:27:23,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T08:27:23,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T08:27:23,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T08:27:23,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T08:27:23,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T08:27:23,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T08:27:23,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T08:27:23,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/nfs.dump.dir in system properties and HBase conf 2024-12-08T08:27:23,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/java.io.tmpdir in system properties and HBase conf 2024-12-08T08:27:23,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T08:27:23,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T08:27:23,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T08:27:24,589 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T08:27:24,661 INFO [Time-limited test {}] log.Log(170): Logging initialized @2742ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T08:27:24,737 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T08:27:24,801 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T08:27:24,827 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T08:27:24,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T08:27:24,830 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T08:27:24,847 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T08:27:24,850 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6193f924{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/hadoop.log.dir/,AVAILABLE} 2024-12-08T08:27:24,851 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@734175dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T08:27:25,054 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bc299d4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/java.io.tmpdir/jetty-localhost-37663-hadoop-hdfs-3_4_1-tests_jar-_-any-1665723437218926715/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T08:27:25,061 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e20720d{HTTP/1.1, (http/1.1)}{localhost:37663} 2024-12-08T08:27:25,061 INFO [Time-limited test {}] server.Server(415): Started @3143ms 2024-12-08T08:27:25,704 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T08:27:25,715 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T08:27:25,717 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T08:27:25,717 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T08:27:25,717 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T08:27:25,718 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c9b6fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/hadoop.log.dir/,AVAILABLE} 2024-12-08T08:27:25,718 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@53038108{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T08:27:25,840 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@729134c5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/java.io.tmpdir/jetty-localhost-34383-hadoop-hdfs-3_4_1-tests_jar-_-any-12602966399728261821/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T08:27:25,841 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e0faa97{HTTP/1.1, (http/1.1)}{localhost:34383} 2024-12-08T08:27:25,841 INFO [Time-limited test {}] server.Server(415): Started @3922ms 2024-12-08T08:27:25,898 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T08:27:26,940 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/data/data1/current/BP-242960261-172.17.0.3-1733646444019/current, will proceed with Du for space computation calculation, 2024-12-08T08:27:26,940 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/data/data2/current/BP-242960261-172.17.0.3-1733646444019/current, will proceed with Du for space computation calculation, 2024-12-08T08:27:26,967 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T08:27:27,019 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6cb0a3c4274326e1 with lease ID 0xf4227725b61e457a: Processing first storage report for DS-cb92d21b-4ff2-45cb-a181-d74bbe865e3c from datanode DatanodeRegistration(127.0.0.1:37723, datanodeUuid=c92ae55d-6ed0-427f-801d-bf23bb444ec7, infoPort=37429, infoSecurePort=0, ipcPort=35843, storageInfo=lv=-57;cid=testClusterID;nsid=1429378041;c=1733646444019) 2024-12-08T08:27:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6cb0a3c4274326e1 with lease ID 0xf4227725b61e457a: from storage DS-cb92d21b-4ff2-45cb-a181-d74bbe865e3c node DatanodeRegistration(127.0.0.1:37723, datanodeUuid=c92ae55d-6ed0-427f-801d-bf23bb444ec7, infoPort=37429, infoSecurePort=0, ipcPort=35843, storageInfo=lv=-57;cid=testClusterID;nsid=1429378041;c=1733646444019), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T08:27:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6cb0a3c4274326e1 with lease ID 0xf4227725b61e457a: Processing first storage report for DS-1ce7ce03-4633-4292-95b1-d2125a001b5e from datanode DatanodeRegistration(127.0.0.1:37723, datanodeUuid=c92ae55d-6ed0-427f-801d-bf23bb444ec7, infoPort=37429, infoSecurePort=0, ipcPort=35843, storageInfo=lv=-57;cid=testClusterID;nsid=1429378041;c=1733646444019) 2024-12-08T08:27:27,021 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6cb0a3c4274326e1 with lease ID 0xf4227725b61e457a: from storage DS-1ce7ce03-4633-4292-95b1-d2125a001b5e node DatanodeRegistration(127.0.0.1:37723, datanodeUuid=c92ae55d-6ed0-427f-801d-bf23bb444ec7, infoPort=37429, infoSecurePort=0, ipcPort=35843, storageInfo=lv=-57;cid=testClusterID;nsid=1429378041;c=1733646444019), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T08:27:27,057 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3 2024-12-08T08:27:27,144 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/zookeeper_0, clientPort=60702, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T08:27:27,158 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60702 2024-12-08T08:27:27,170 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:27:27,175 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:27:27,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741825_1001 (size=7) 2024-12-08T08:27:27,836 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 with version=8 2024-12-08T08:27:27,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/hbase-staging 2024-12-08T08:27:27,921 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T08:27:28,195 INFO [Time-limited test {}] client.ConnectionUtils(128): master/422e41b75614:0 server-side Connection retries=9 2024-12-08T08:27:28,206 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:27:28,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T08:27:28,212 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T08:27:28,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:27:28,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T08:27:28,353 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T08:27:28,418 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T08:27:28,428 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T08:27:28,432 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T08:27:28,458 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 88141 (auto-detected) 2024-12-08T08:27:28,459 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-08T08:27:28,478 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33175 2024-12-08T08:27:28,499 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33175 connecting to ZooKeeper ensemble=127.0.0.1:60702 2024-12-08T08:27:28,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:331750x0, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T08:27:28,621 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33175-0x100048978dd0000 connected 2024-12-08T08:27:28,712 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:27:28,714 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:27:28,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:27:28,727 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4, hbase.cluster.distributed=false 2024-12-08T08:27:28,749 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T08:27:28,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33175 2024-12-08T08:27:28,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33175 2024-12-08T08:27:28,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33175 2024-12-08T08:27:28,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33175 2024-12-08T08:27:28,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33175 2024-12-08T08:27:28,870 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/422e41b75614:0 server-side Connection retries=9 2024-12-08T08:27:28,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:27:28,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T08:27:28,872 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T08:27:28,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T08:27:28,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T08:27:28,875 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T08:27:28,877 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T08:27:28,878 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32911 2024-12-08T08:27:28,879 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32911 connecting to ZooKeeper ensemble=127.0.0.1:60702 2024-12-08T08:27:28,881 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:27:28,884 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:27:28,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329110x0, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T08:27:28,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32911-0x100048978dd0001 connected 2024-12-08T08:27:28,901 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:27:28,905 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T08:27:28,913 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T08:27:28,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T08:27:28,921 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T08:27:28,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32911 2024-12-08T08:27:28,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32911 2024-12-08T08:27:28,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32911 2024-12-08T08:27:28,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32911 2024-12-08T08:27:28,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32911 2024-12-08T08:27:28,942 DEBUG [M:0;422e41b75614:33175 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;422e41b75614:33175 2024-12-08T08:27:28,943 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/422e41b75614,33175,1733646448021 2024-12-08T08:27:28,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:27:28,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:27:28,955 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/422e41b75614,33175,1733646448021 2024-12-08T08:27:28,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:28,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T08:27:28,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:28,985 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T08:27:28,986 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/422e41b75614,33175,1733646448021 from backup master directory 2024-12-08T08:27:28,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/422e41b75614,33175,1733646448021 2024-12-08T08:27:28,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:27:28,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T08:27:28,995 WARN [master/422e41b75614:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T08:27:28,995 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=422e41b75614,33175,1733646448021 2024-12-08T08:27:28,998 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T08:27:28,999 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T08:27:29,056 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/hbase.id] with ID: eab703f8-51cc-4200-b903-b057e2d90e5f 2024-12-08T08:27:29,056 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/.tmp/hbase.id 2024-12-08T08:27:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741826_1002 (size=42) 2024-12-08T08:27:29,471 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/.tmp/hbase.id]:[hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/hbase.id] 2024-12-08T08:27:29,520 INFO [master/422e41b75614:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T08:27:29,524 INFO [master/422e41b75614:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T08:27:29,540 INFO [master/422e41b75614:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-08T08:27:29,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:29,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:29,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741827_1003 (size=196) 2024-12-08T08:27:30,026 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:27:30,029 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T08:27:30,051 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:27:30,055 INFO [master/422e41b75614:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T08:27:30,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741828_1004 (size=1189) 2024-12-08T08:27:30,510 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store 2024-12-08T08:27:30,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741829_1005 (size=34) 2024-12-08T08:27:30,936 INFO [master/422e41b75614:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T08:27:30,940 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:27:30,941 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T08:27:30,941 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:27:30,941 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:27:30,943 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T08:27:30,943 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:27:30,943 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:27:30,944 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733646450941Disabling compacts and flushes for region at 1733646450941Disabling writes for close at 1733646450943 (+2 ms)Writing region close event to WAL at 1733646450943Closed at 1733646450943 2024-12-08T08:27:30,946 WARN [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/.initializing 2024-12-08T08:27:30,946 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/WALs/422e41b75614,33175,1733646448021 2024-12-08T08:27:30,953 INFO [master/422e41b75614:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T08:27:30,967 INFO [master/422e41b75614:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=422e41b75614%2C33175%2C1733646448021, suffix=, logDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/WALs/422e41b75614,33175,1733646448021, archiveDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/oldWALs, maxLogs=10 2024-12-08T08:27:30,987 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/WALs/422e41b75614,33175,1733646448021/422e41b75614%2C33175%2C1733646448021.1733646450972, exclude list is [], retry=0 2024-12-08T08:27:31,001 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37723,DS-cb92d21b-4ff2-45cb-a181-d74bbe865e3c,DISK] 2024-12-08T08:27:31,004 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T08:27:31,035 INFO [master/422e41b75614:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/WALs/422e41b75614,33175,1733646448021/422e41b75614%2C33175%2C1733646448021.1733646450972 2024-12-08T08:27:31,036 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37429:37429)] 2024-12-08T08:27:31,037 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:27:31,037 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:27:31,040 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,042 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T08:27:31,117 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:31,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:27:31,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T08:27:31,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:31,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:27:31,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T08:27:31,133 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:31,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:27:31,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T08:27:31,139 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:31,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:27:31,142 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,145 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,147 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,153 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,153 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,157 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T08:27:31,162 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T08:27:31,167 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:27:31,168 INFO [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71530333, jitterRate=0.0658850222826004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T08:27:31,174 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733646451054Initializing all the Stores at 1733646451056 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733646451057 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646451057Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646451058 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646451058Cleaning up temporary data from old regions at 1733646451153 (+95 ms)Region opened successfully at 1733646451174 (+21 ms) 2024-12-08T08:27:31,175 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T08:27:31,210 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11975525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=422e41b75614/172.17.0.3:0 2024-12-08T08:27:31,246 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T08:27:31,257 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T08:27:31,257 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T08:27:31,260 INFO [master/422e41b75614:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T08:27:31,261 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T08:27:31,266 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-08T08:27:31,266 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T08:27:31,292 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T08:27:31,302 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T08:27:31,352 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T08:27:31,355 INFO [master/422e41b75614:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T08:27:31,357 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T08:27:31,363 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T08:27:31,365 INFO [master/422e41b75614:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T08:27:31,369 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T08:27:31,382 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T08:27:31,384 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T08:27:31,394 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T08:27:31,415 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T08:27:31,426 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T08:27:31,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T08:27:31,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T08:27:31,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:31,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:31,439 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=422e41b75614,33175,1733646448021, sessionid=0x100048978dd0000, setting cluster-up flag (Was=false) 2024-12-08T08:27:31,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:31,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:31,500 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T08:27:31,506 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=422e41b75614,33175,1733646448021 2024-12-08T08:27:31,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:31,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:27:31,563 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T08:27:31,568 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=422e41b75614,33175,1733646448021 2024-12-08T08:27:31,578 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T08:27:31,633 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(746): ClusterId : eab703f8-51cc-4200-b903-b057e2d90e5f 2024-12-08T08:27:31,635 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T08:27:31,644 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T08:27:31,649 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T08:27:31,649 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T08:27:31,652 INFO [master/422e41b75614:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T08:27:31,657 INFO [master/422e41b75614:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T08:27:31,659 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T08:27:31,659 DEBUG [RS:0;422e41b75614:32911 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a35f74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=422e41b75614/172.17.0.3:0 2024-12-08T08:27:31,662 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 422e41b75614,33175,1733646448021 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T08:27:31,669 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:27:31,669 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:27:31,669 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:27:31,669 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/422e41b75614:0, corePoolSize=5, maxPoolSize=5 2024-12-08T08:27:31,669 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/422e41b75614:0, corePoolSize=10, maxPoolSize=10 2024-12-08T08:27:31,670 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,670 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/422e41b75614:0, corePoolSize=2, maxPoolSize=2 2024-12-08T08:27:31,670 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,671 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733646481671 2024-12-08T08:27:31,672 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T08:27:31,673 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T08:27:31,675 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T08:27:31,675 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;422e41b75614:32911 2024-12-08T08:27:31,676 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T08:27:31,676 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T08:27:31,677 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T08:27:31,677 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T08:27:31,677 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T08:27:31,679 INFO [RS:0;422e41b75614:32911 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T08:27:31,678 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,679 INFO [RS:0;422e41b75614:32911 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T08:27:31,680 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T08:27:31,681 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T08:27:31,681 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:31,682 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T08:27:31,682 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T08:27:31,683 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T08:27:31,683 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(2659): reportForDuty to master=422e41b75614,33175,1733646448021 with port=32911, startcode=1733646448831 2024-12-08T08:27:31,685 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T08:27:31,685 INFO [master/422e41b75614:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T08:27:31,687 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.large.0-1733646451686,5,FailOnTimeoutGroup] 2024-12-08T08:27:31,687 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.small.0-1733646451687,5,FailOnTimeoutGroup] 2024-12-08T08:27:31,687 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,687 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T08:27:31,688 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,689 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741831_1007 (size=1321) 2024-12-08T08:27:31,694 DEBUG [RS:0;422e41b75614:32911 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T08:27:31,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44645, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T08:27:31,724 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33175 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 422e41b75614,32911,1733646448831 2024-12-08T08:27:31,726 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33175 {}] master.ServerManager(517): Registering regionserver=422e41b75614,32911,1733646448831 2024-12-08T08:27:31,738 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:27:31,739 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33261 2024-12-08T08:27:31,739 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T08:27:31,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T08:27:31,753 DEBUG [RS:0;422e41b75614:32911 {}] zookeeper.ZKUtil(111): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/422e41b75614,32911,1733646448831 2024-12-08T08:27:31,753 WARN [RS:0;422e41b75614:32911 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T08:27:31,753 INFO [RS:0;422e41b75614:32911 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T08:27:31,754 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/WALs/422e41b75614,32911,1733646448831 2024-12-08T08:27:31,756 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [422e41b75614,32911,1733646448831] 2024-12-08T08:27:31,787 INFO [RS:0;422e41b75614:32911 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T08:27:31,798 INFO [RS:0;422e41b75614:32911 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T08:27:31,802 INFO [RS:0;422e41b75614:32911 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T08:27:31,802 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,803 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T08:27:31,808 INFO [RS:0;422e41b75614:32911 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T08:27:31,809 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,809 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,809 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,809 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,809 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,809 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/422e41b75614:0, corePoolSize=2, maxPoolSize=2 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/422e41b75614:0, corePoolSize=1, maxPoolSize=1 2024-12-08T08:27:31,810 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/422e41b75614:0, corePoolSize=3, maxPoolSize=3 2024-12-08T08:27:31,811 DEBUG [RS:0;422e41b75614:32911 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/422e41b75614:0, corePoolSize=3, maxPoolSize=3 2024-12-08T08:27:31,811 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,811 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,811 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,812 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,812 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,812 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,32911,1733646448831-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T08:27:31,826 INFO [RS:0;422e41b75614:32911 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T08:27:31,827 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,32911,1733646448831-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,827 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,828 INFO [RS:0;422e41b75614:32911 {}] regionserver.Replication(171): 422e41b75614,32911,1733646448831 started 2024-12-08T08:27:31,842 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:27:31,842 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1482): Serving as 422e41b75614,32911,1733646448831, RpcServer on 422e41b75614/172.17.0.3:32911, sessionid=0x100048978dd0001 2024-12-08T08:27:31,843 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T08:27:31,843 DEBUG [RS:0;422e41b75614:32911 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 422e41b75614,32911,1733646448831 2024-12-08T08:27:31,843 DEBUG [RS:0;422e41b75614:32911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '422e41b75614,32911,1733646448831' 2024-12-08T08:27:31,843 DEBUG [RS:0;422e41b75614:32911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T08:27:31,844 DEBUG [RS:0;422e41b75614:32911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T08:27:31,845 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T08:27:31,845 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T08:27:31,845 DEBUG [RS:0;422e41b75614:32911 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 422e41b75614,32911,1733646448831 2024-12-08T08:27:31,845 DEBUG [RS:0;422e41b75614:32911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '422e41b75614,32911,1733646448831' 2024-12-08T08:27:31,845 DEBUG [RS:0;422e41b75614:32911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T08:27:31,846 DEBUG [RS:0;422e41b75614:32911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T08:27:31,847 DEBUG [RS:0;422e41b75614:32911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T08:27:31,847 INFO [RS:0;422e41b75614:32911 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T08:27:31,847 INFO [RS:0;422e41b75614:32911 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T08:27:32,093 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T08:27:32,094 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:27:32,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741832_1008 (size=32) 2024-12-08T08:27:32,507 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:27:32,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T08:27:32,515 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T08:27:32,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:32,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:27:32,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T08:27:32,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T08:27:32,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:32,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:27:32,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T08:27:32,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T08:27:32,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:32,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:27:32,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T08:27:32,530 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T08:27:32,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:27:32,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:27:32,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T08:27:32,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740 2024-12-08T08:27:32,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740 2024-12-08T08:27:32,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T08:27:32,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T08:27:32,538 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T08:27:32,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T08:27:32,545 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:27:32,546 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74400263, jitterRate=0.1086503118276596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T08:27:32,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733646452508Initializing all the Stores at 1733646452511 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733646452511Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733646452511Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646452511Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733646452511Cleaning up temporary data from old regions at 1733646452537 (+26 ms)Region opened successfully at 1733646452547 (+10 ms) 2024-12-08T08:27:32,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T08:27:32,548 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T08:27:32,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T08:27:32,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T08:27:32,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T08:27:32,549 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T08:27:32,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733646452548Disabling compacts and flushes for region at 1733646452548Disabling writes for close at 1733646452548Writing region close event to WAL at 1733646452549 (+1 ms)Closed at 1733646452549 2024-12-08T08:27:32,552 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T08:27:32,552 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T08:27:32,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T08:27:32,564 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T08:27:32,566 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T08:27:32,718 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:33,719 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:34,720 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:35,721 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:36,721 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:37,722 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:38,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T08:27:38,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T08:27:38,723 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:39,724 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:40,725 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:41,725 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:42,726 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:43,727 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:44,728 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:45,729 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:46,729 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:47,730 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:48,731 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:49,732 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:50,733 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:51,734 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:52,735 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:53,736 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:54,738 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:55,739 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:56,740 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:57,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:27:57,741 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:58,742 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:27:59,743 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:28:00,744 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:28:01,745 WARN [422e41b75614:33175 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T08:28:01,860 INFO [RS:0;422e41b75614:32911 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T08:28:01,864 INFO [RS:0;422e41b75614:32911 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=422e41b75614%2C32911%2C1733646448831, suffix=, logDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/WALs/422e41b75614,32911,1733646448831, archiveDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/oldWALs, maxLogs=32 2024-12-08T08:28:01,877 DEBUG [RS:0;422e41b75614:32911 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/WALs/422e41b75614,32911,1733646448831/422e41b75614%2C32911%2C1733646448831.1733646481866, exclude list is [], retry=0 2024-12-08T08:28:01,881 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37723,DS-cb92d21b-4ff2-45cb-a181-d74bbe865e3c,DISK] 2024-12-08T08:28:01,884 INFO [RS:0;422e41b75614:32911 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/WALs/422e41b75614,32911,1733646448831/422e41b75614%2C32911%2C1733646448831.1733646481866 2024-12-08T08:28:01,885 DEBUG [RS:0;422e41b75614:32911 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37429:37429)] 2024-12-08T08:28:02,000 DEBUG [422e41b75614:33175 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T08:28:02,015 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:02,021 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 422e41b75614,32911,1733646448831, state=OPENING 2024-12-08T08:28:02,098 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T08:28:02,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:28:02,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:28:02,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:28:02,112 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:28:02,115 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T08:28:02,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:02,296 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T08:28:02,299 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52071, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T08:28:02,310 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T08:28:02,311 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T08:28:02,311 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T08:28:02,315 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=422e41b75614%2C32911%2C1733646448831.meta, suffix=.meta, logDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/WALs/422e41b75614,32911,1733646448831, archiveDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/oldWALs, maxLogs=32 2024-12-08T08:28:02,327 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/WALs/422e41b75614,32911,1733646448831/422e41b75614%2C32911%2C1733646448831.meta.1733646482316.meta, exclude list is [], retry=0 2024-12-08T08:28:02,331 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37723,DS-cb92d21b-4ff2-45cb-a181-d74bbe865e3c,DISK] 2024-12-08T08:28:02,334 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/WALs/422e41b75614,32911,1733646448831/422e41b75614%2C32911%2C1733646448831.meta.1733646482316.meta 2024-12-08T08:28:02,334 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37429:37429)] 2024-12-08T08:28:02,335 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:02,336 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T08:28:02,338 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T08:28:02,342 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T08:28:02,346 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T08:28:02,346 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:02,346 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T08:28:02,347 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T08:28:02,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T08:28:02,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T08:28:02,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:02,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:28:02,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T08:28:02,354 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T08:28:02,354 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:02,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:28:02,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T08:28:02,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T08:28:02,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:02,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:28:02,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T08:28:02,360 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T08:28:02,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:02,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T08:28:02,361 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T08:28:02,363 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740 2024-12-08T08:28:02,366 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740 2024-12-08T08:28:02,368 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T08:28:02,368 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T08:28:02,369 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T08:28:02,371 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T08:28:02,373 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67476109, jitterRate=0.0054723769426345825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T08:28:02,373 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T08:28:02,374 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733646482347Writing region info on filesystem at 1733646482347Initializing all the Stores at 1733646482349 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733646482349Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733646482349Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646482349Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733646482349Cleaning up temporary data from old regions at 1733646482368 (+19 ms)Running coprocessor post-open hooks at 1733646482373 (+5 ms)Region opened successfully at 1733646482374 (+1 ms) 2024-12-08T08:28:02,380 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733646482288 2024-12-08T08:28:02,390 DEBUG [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T08:28:02,391 INFO [RS_OPEN_META-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T08:28:02,392 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:02,395 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 422e41b75614,32911,1733646448831, state=OPEN 2024-12-08T08:28:02,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T08:28:02,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T08:28:02,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:28:02,432 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T08:28:02,433 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=422e41b75614,32911,1733646448831 2024-12-08T08:28:02,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T08:28:02,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=422e41b75614,32911,1733646448831 in 316 msec 2024-12-08T08:28:02,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T08:28:02,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8830 sec 2024-12-08T08:28:02,447 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T08:28:02,447 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T08:28:02,464 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:28:02,465 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,32911,1733646448831, seqNum=-1] 2024-12-08T08:28:02,483 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:28:02,485 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44235, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:28:02,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9000 sec 2024-12-08T08:28:02,506 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733646482506, completionTime=-1 2024-12-08T08:28:02,509 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T08:28:02,509 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T08:28:02,535 INFO [master/422e41b75614:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T08:28:02,536 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733646542535 2024-12-08T08:28:02,536 INFO [master/422e41b75614:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733646602536 2024-12-08T08:28:02,536 INFO [master/422e41b75614:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-08T08:28:02,537 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,33175,1733646448021-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T08:28:02,538 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,33175,1733646448021-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:28:02,538 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,33175,1733646448021-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:28:02,539 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-422e41b75614:33175, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:28:02,540 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T08:28:02,540 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T08:28:02,547 DEBUG [master/422e41b75614:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T08:28:02,569 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.573sec 2024-12-08T08:28:02,570 INFO [master/422e41b75614:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T08:28:02,572 INFO [master/422e41b75614:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T08:28:02,573 INFO [master/422e41b75614:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T08:28:02,573 INFO [master/422e41b75614:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T08:28:02,574 INFO [master/422e41b75614:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T08:28:02,575 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,33175,1733646448021-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T08:28:02,575 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,33175,1733646448021-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T08:28:02,583 DEBUG [master/422e41b75614:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T08:28:02,584 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T08:28:02,584 INFO [master/422e41b75614:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=422e41b75614,33175,1733646448021-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T08:28:02,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d127fb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:28:02,681 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T08:28:02,681 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T08:28:02,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,33175,-1 for getting cluster id 2024-12-08T08:28:02,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:28:02,711 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'eab703f8-51cc-4200-b903-b057e2d90e5f' 2024-12-08T08:28:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:28:02,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "eab703f8-51cc-4200-b903-b057e2d90e5f" 2024-12-08T08:28:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21cd6197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:28:02,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,33175,-1] 2024-12-08T08:28:02,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:28:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:28:02,722 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:28:02,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ae6fd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:28:02,728 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:28:02,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,32911,1733646448831, seqNum=-1] 2024-12-08T08:28:02,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:28:02,739 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:28:02,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=422e41b75614,33175,1733646448021 2024-12-08T08:28:02,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189b4978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:28:02,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 422e41b75614,33175,-1 for getting cluster id 2024-12-08T08:28:02,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T08:28:02,758 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'eab703f8-51cc-4200-b903-b057e2d90e5f' 2024-12-08T08:28:02,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T08:28:02,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "eab703f8-51cc-4200-b903-b057e2d90e5f" 2024-12-08T08:28:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217979fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:28:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [422e41b75614,33175,-1] 2024-12-08T08:28:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T08:28:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:28:02,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T08:28:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8afe79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T08:28:02,774 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=245, ProcessCount=11, AvailableMemoryMB=7461 2024-12-08T08:28:02,782 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T08:28:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 422e41b75614,33175,1733646448021 2024-12-08T08:28:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48dd1ca4 2024-12-08T08:28:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T08:28:02,794 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T08:28:02,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-08T08:28:02,811 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:02,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-08T08:28:02,813 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:02,815 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:28:02,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741835_1011 (size=428) 2024-12-08T08:28:03,234 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07499732554247c1fe88f68b036dfcc1, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:03,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741836_1012 (size=79) 2024-12-08T08:28:03,649 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:03,649 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 07499732554247c1fe88f68b036dfcc1, disabling compactions & flushes 2024-12-08T08:28:03,649 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:28:03,650 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:28:03,650 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. after waiting 0 ms 2024-12-08T08:28:03,650 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:28:03,650 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:28:03,650 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07499732554247c1fe88f68b036dfcc1: Waiting for close lock at 1733646483649Disabling compacts and flushes for region at 1733646483649Disabling writes for close at 1733646483650 (+1 ms)Writing region close event to WAL at 1733646483650Closed at 1733646483650 2024-12-08T08:28:03,655 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:03,662 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733646483655"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646483655"}]},"ts":"1733646483655"} 2024-12-08T08:28:03,667 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:03,669 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:03,672 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646483669"}]},"ts":"1733646483669"} 2024-12-08T08:28:03,676 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-08T08:28:03,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=07499732554247c1fe88f68b036dfcc1, ASSIGN}] 2024-12-08T08:28:03,680 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=07499732554247c1fe88f68b036dfcc1, ASSIGN 2024-12-08T08:28:03,681 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=07499732554247c1fe88f68b036dfcc1, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:28:03,834 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=07499732554247c1fe88f68b036dfcc1, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:03,841 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=07499732554247c1fe88f68b036dfcc1, ASSIGN because future has completed 2024-12-08T08:28:03,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07499732554247c1fe88f68b036dfcc1, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:04,013 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:28:04,013 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 07499732554247c1fe88f68b036dfcc1, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:04,013 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,013 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:04,014 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,014 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,016 INFO [StoreOpener-07499732554247c1fe88f68b036dfcc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,018 INFO [StoreOpener-07499732554247c1fe88f68b036dfcc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07499732554247c1fe88f68b036dfcc1 columnFamilyName testFamily 2024-12-08T08:28:04,018 DEBUG [StoreOpener-07499732554247c1fe88f68b036dfcc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:04,019 INFO [StoreOpener-07499732554247c1fe88f68b036dfcc1-1 {}] regionserver.HStore(327): Store=07499732554247c1fe88f68b036dfcc1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:04,020 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,021 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,022 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,022 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,022 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,025 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,028 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:04,029 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 07499732554247c1fe88f68b036dfcc1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58851852, jitterRate=-0.12303906679153442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:04,029 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:28:04,030 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 07499732554247c1fe88f68b036dfcc1: Running coprocessor pre-open hook at 1733646484014Writing region info on filesystem at 1733646484014Initializing all the Stores at 1733646484016 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646484016Cleaning up temporary data from old regions at 1733646484022 (+6 ms)Running coprocessor post-open hooks at 1733646484029 (+7 ms)Region opened successfully at 1733646484030 (+1 ms) 2024-12-08T08:28:04,031 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1., pid=6, masterSystemTime=1733646483999 2024-12-08T08:28:04,035 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:28:04,035 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:28:04,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=07499732554247c1fe88f68b036dfcc1, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:04,039 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07499732554247c1fe88f68b036dfcc1, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:04,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T08:28:04,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 07499732554247c1fe88f68b036dfcc1, server=422e41b75614,32911,1733646448831 in 199 msec 2024-12-08T08:28:04,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T08:28:04,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=07499732554247c1fe88f68b036dfcc1, ASSIGN in 368 msec 2024-12-08T08:28:04,050 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:04,050 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646484050"}]},"ts":"1733646484050"} 2024-12-08T08:28:04,054 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-08T08:28:04,056 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:04,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2530 sec 2024-12-08T08:28:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T08:28:05,867 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-08T08:28:05,867 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-08T08:28:05,870 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:05,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-08T08:28:05,881 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:05,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-08T08:28:05,884 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T08:28:05,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=422e41b75614,32911,1733646448831, seqNum=-1] 2024-12-08T08:28:05,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T08:28:05,888 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T08:28:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:05,906 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:05,909 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-08T08:28:05,915 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-08T08:28:05,920 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-08T08:28:05,922 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-08T08:28:05,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=583559152127705096 2024-12-08T08:28:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646488921, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:05,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-08T08:28:05,950 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-08T08:28:05,951 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:05,953 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:05,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=583559152127705098 2024-12-08T08:28:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646488952, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:05,957 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:05,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=583559152127705099 2024-12-08T08:28:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646488956, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:05,960 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:05,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=583559152127705100 2024-12-08T08:28:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646488960, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:05,965 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:05,975 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/422e41b75614:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2141993544_22 at /127.0.0.1:46958 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=226 (was 245), ProcessCount=11 (was 11), AvailableMemoryMB=7450 (was 7461) 2024-12-08T08:28:05,982 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=226, ProcessCount=11, AvailableMemoryMB=7450 2024-12-08T08:28:05,985 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:05,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-08T08:28:05,989 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:05,989 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:05,989 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-08T08:28:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:28:05,991 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:05,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741837_1013 (size=414) 2024-12-08T08:28:06,407 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 15b9440e89d0596da15af951d1128587, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:06,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741838_1014 (size=65) 2024-12-08T08:28:06,819 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:06,819 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 15b9440e89d0596da15af951d1128587, disabling compactions & flushes 2024-12-08T08:28:06,820 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:28:06,820 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:28:06,820 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. after waiting 0 ms 2024-12-08T08:28:06,820 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:28:06,820 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:28:06,820 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 15b9440e89d0596da15af951d1128587: Waiting for close lock at 1733646486819Disabling compacts and flushes for region at 1733646486819Disabling writes for close at 1733646486820 (+1 ms)Writing region close event to WAL at 1733646486820Closed at 1733646486820 2024-12-08T08:28:06,824 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:06,824 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733646486824"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646486824"}]},"ts":"1733646486824"} 2024-12-08T08:28:06,829 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:06,832 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:06,832 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646486832"}]},"ts":"1733646486832"} 2024-12-08T08:28:06,836 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-08T08:28:06,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=15b9440e89d0596da15af951d1128587, ASSIGN}] 2024-12-08T08:28:06,838 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=15b9440e89d0596da15af951d1128587, ASSIGN 2024-12-08T08:28:06,840 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=15b9440e89d0596da15af951d1128587, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:06,992 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=15b9440e89d0596da15af951d1128587, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:06,999 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=15b9440e89d0596da15af951d1128587, ASSIGN because future has completed 2024-12-08T08:28:07,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15b9440e89d0596da15af951d1128587, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:28:07,168 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:28:07,168 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 15b9440e89d0596da15af951d1128587, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:07,168 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,168 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:07,169 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,169 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,171 INFO [StoreOpener-15b9440e89d0596da15af951d1128587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,174 INFO [StoreOpener-15b9440e89d0596da15af951d1128587-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15b9440e89d0596da15af951d1128587 columnFamilyName testFamily 2024-12-08T08:28:07,174 DEBUG [StoreOpener-15b9440e89d0596da15af951d1128587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:07,175 INFO [StoreOpener-15b9440e89d0596da15af951d1128587-1 {}] regionserver.HStore(327): Store=15b9440e89d0596da15af951d1128587/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:07,176 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,178 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,178 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,179 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,179 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,184 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:07,189 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 15b9440e89d0596da15af951d1128587; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62706130, jitterRate=-0.06560584902763367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:07,189 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15b9440e89d0596da15af951d1128587 2024-12-08T08:28:07,191 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 15b9440e89d0596da15af951d1128587: Running coprocessor pre-open hook at 1733646487169Writing region info on filesystem at 1733646487169Initializing all the Stores at 1733646487171 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646487171Cleaning up temporary data from old regions at 1733646487179 (+8 ms)Running coprocessor post-open hooks at 1733646487189 (+10 ms)Region opened successfully at 1733646487191 (+2 ms) 2024-12-08T08:28:07,192 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., pid=9, masterSystemTime=1733646487155 2024-12-08T08:28:07,196 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:28:07,196 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:28:07,197 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=15b9440e89d0596da15af951d1128587, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:07,200 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15b9440e89d0596da15af951d1128587, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:07,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-08T08:28:07,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 15b9440e89d0596da15af951d1128587, server=422e41b75614,32911,1733646448831 in 202 msec 2024-12-08T08:28:07,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T08:28:07,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=15b9440e89d0596da15af951d1128587, ASSIGN in 370 msec 2024-12-08T08:28:07,213 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:07,213 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646487213"}]},"ts":"1733646487213"} 2024-12-08T08:28:07,216 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-08T08:28:07,217 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:07,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2320 sec 2024-12-08T08:28:07,882 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:28:07,906 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-08T08:28:07,907 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T08:28:07,907 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-08T08:28:08,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T08:28:08,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:08,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T08:28:08,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T08:28:08,418 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-08T08:28:08,418 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:08,420 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-08T08:28:08,420 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T08:28:09,035 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-08T08:28:09,035 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-08T08:28:09,035 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:09,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-08T08:28:09,043 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:09,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-08T08:28:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:09,056 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:09,057 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-08T08:28:09,060 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T08:28:09,062 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-08T08:28:15,072 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., hostname=422e41b75614,32911,1733646448831, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., hostname=422e41b75614,32911,1733646448831, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=422e41b75614:32911 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-12-08T08:28:15,073 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., hostname=422e41b75614,32911,1733646448831, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-12-08T08:28:15,073 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., hostname=422e41b75614,32911,1733646448831, seqNum=2 from cache 2024-12-08T08:28:15,082 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=207 (was 226), ProcessCount=11 (was 11), AvailableMemoryMB=8519 (was 7450) - AvailableMemoryMB LEAK? - 2024-12-08T08:28:15,093 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=207, ProcessCount=11, AvailableMemoryMB=8518 2024-12-08T08:28:15,096 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:15,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-08T08:28:15,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:15,100 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:15,101 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-08T08:28:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-08T08:28:15,102 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:15,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741839_1015 (size=412) 2024-12-08T08:28:15,513 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => efb3a6942cdf755b7102cf2ed271587b, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741840_1016 (size=63) 2024-12-08T08:28:15,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-08T08:28:15,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:15,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing efb3a6942cdf755b7102cf2ed271587b, disabling compactions & flushes 2024-12-08T08:28:15,924 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:28:15,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:28:15,924 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. after waiting 0 ms 2024-12-08T08:28:15,925 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:28:15,925 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:28:15,925 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for efb3a6942cdf755b7102cf2ed271587b: Waiting for close lock at 1733646495924Disabling compacts and flushes for region at 1733646495924Disabling writes for close at 1733646495924Writing region close event to WAL at 1733646495925 (+1 ms)Closed at 1733646495925 2024-12-08T08:28:15,927 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:15,928 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733646495927"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646495927"}]},"ts":"1733646495927"} 2024-12-08T08:28:15,932 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:15,934 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:15,935 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646495934"}]},"ts":"1733646495934"} 2024-12-08T08:28:15,939 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-08T08:28:15,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=efb3a6942cdf755b7102cf2ed271587b, ASSIGN}] 2024-12-08T08:28:15,942 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=efb3a6942cdf755b7102cf2ed271587b, ASSIGN 2024-12-08T08:28:15,945 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=efb3a6942cdf755b7102cf2ed271587b, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:16,096 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=efb3a6942cdf755b7102cf2ed271587b, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:16,101 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=efb3a6942cdf755b7102cf2ed271587b, ASSIGN because future has completed 2024-12-08T08:28:16,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure efb3a6942cdf755b7102cf2ed271587b, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-08T08:28:16,264 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:28:16,265 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => efb3a6942cdf755b7102cf2ed271587b, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:16,265 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,265 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:16,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,268 INFO [StoreOpener-efb3a6942cdf755b7102cf2ed271587b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,271 INFO [StoreOpener-efb3a6942cdf755b7102cf2ed271587b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efb3a6942cdf755b7102cf2ed271587b columnFamilyName testFamily 2024-12-08T08:28:16,271 DEBUG [StoreOpener-efb3a6942cdf755b7102cf2ed271587b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:16,272 INFO [StoreOpener-efb3a6942cdf755b7102cf2ed271587b-1 {}] regionserver.HStore(327): Store=efb3a6942cdf755b7102cf2ed271587b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:16,272 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,275 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,280 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,284 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:16,285 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened efb3a6942cdf755b7102cf2ed271587b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73922443, jitterRate=0.10153023898601532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:16,285 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:28:16,286 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for efb3a6942cdf755b7102cf2ed271587b: Running coprocessor pre-open hook at 1733646496266Writing region info on filesystem at 1733646496266Initializing all the Stores at 1733646496267 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646496267Cleaning up temporary data from old regions at 1733646496276 (+9 ms)Running coprocessor post-open hooks at 1733646496285 (+9 ms)Region opened successfully at 1733646496286 (+1 ms) 2024-12-08T08:28:16,288 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b., pid=12, masterSystemTime=1733646496257 2024-12-08T08:28:16,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:28:16,291 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:28:16,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=efb3a6942cdf755b7102cf2ed271587b, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:16,296 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure efb3a6942cdf755b7102cf2ed271587b, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:16,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T08:28:16,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure efb3a6942cdf755b7102cf2ed271587b, server=422e41b75614,32911,1733646448831 in 197 msec 2024-12-08T08:28:16,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-08T08:28:16,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=efb3a6942cdf755b7102cf2ed271587b, ASSIGN in 365 msec 2024-12-08T08:28:16,311 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:16,311 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646496311"}]},"ts":"1733646496311"} 2024-12-08T08:28:16,315 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-08T08:28:16,318 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:16,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2220 sec 2024-12-08T08:28:18,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-08T08:28:18,143 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-08T08:28:18,143 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-08T08:28:18,143 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:18,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-08T08:28:18,148 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:18,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-08T08:28:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:18,168 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:18,168 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-08T08:28:18,174 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T08:28:18,177 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T08:28:18,180 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T08:28:18,181 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-08T08:28:18,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-08T08:28:18,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:19,909 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-08T08:28:21,811 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:28:24,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32911 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705109' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T08:28:24,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46652 deadline: 1733646510188, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705109' does not exist 2024-12-08T08:28:24,192 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 422e41b75614,32911,1733646448831 for closing scanner id = 583559152127705109 for efb3a6942cdf755b7102cf2ed271587b of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705109' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:28:24,194 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 207) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7955 (was 8518) 2024-12-08T08:28:24,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=7955 2024-12-08T08:28:24,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-08T08:28:24,209 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:24,209 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:24,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-08T08:28:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T08:28:24,210 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:24,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741841_1017 (size=423) 2024-12-08T08:28:24,623 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f82c14f86610fc5e8020f4821379cdbd, NAME => 'testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:24,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741842_1018 (size=74) 2024-12-08T08:28:24,674 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:24,674 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing f82c14f86610fc5e8020f4821379cdbd, disabling compactions & flushes 2024-12-08T08:28:24,674 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:28:24,674 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:28:24,674 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. after waiting 0 ms 2024-12-08T08:28:24,675 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:28:24,675 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:28:24,675 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for f82c14f86610fc5e8020f4821379cdbd: Waiting for close lock at 1733646504674Disabling compacts and flushes for region at 1733646504674Disabling writes for close at 1733646504674Writing region close event to WAL at 1733646504675 (+1 ms)Closed at 1733646504675 2024-12-08T08:28:24,678 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:24,678 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733646504678"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646504678"}]},"ts":"1733646504678"} 2024-12-08T08:28:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-08T08:28:24,690 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:24,693 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:24,693 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646504693"}]},"ts":"1733646504693"} 2024-12-08T08:28:24,706 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-08T08:28:24,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f82c14f86610fc5e8020f4821379cdbd, ASSIGN}] 2024-12-08T08:28:24,709 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f82c14f86610fc5e8020f4821379cdbd, ASSIGN 2024-12-08T08:28:24,712 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f82c14f86610fc5e8020f4821379cdbd, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:24,863 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f82c14f86610fc5e8020f4821379cdbd, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:24,869 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f82c14f86610fc5e8020f4821379cdbd, ASSIGN because future has completed 2024-12-08T08:28:24,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f82c14f86610fc5e8020f4821379cdbd, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:25,048 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:28:25,048 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => f82c14f86610fc5e8020f4821379cdbd, NAME => 'testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:25,049 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,049 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:25,049 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,049 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,054 INFO [StoreOpener-f82c14f86610fc5e8020f4821379cdbd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,077 INFO [StoreOpener-f82c14f86610fc5e8020f4821379cdbd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f82c14f86610fc5e8020f4821379cdbd columnFamilyName testFamily 2024-12-08T08:28:25,078 DEBUG [StoreOpener-f82c14f86610fc5e8020f4821379cdbd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:25,080 INFO [StoreOpener-f82c14f86610fc5e8020f4821379cdbd-1 {}] regionserver.HStore(327): Store=f82c14f86610fc5e8020f4821379cdbd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:25,081 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,083 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,084 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,085 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,085 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,089 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,101 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:25,103 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened f82c14f86610fc5e8020f4821379cdbd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62967856, jitterRate=-0.061705827713012695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:25,103 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:28:25,104 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for f82c14f86610fc5e8020f4821379cdbd: Running coprocessor pre-open hook at 1733646505050Writing region info on filesystem at 1733646505050Initializing all the Stores at 1733646505053 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646505053Cleaning up temporary data from old regions at 1733646505085 (+32 ms)Running coprocessor post-open hooks at 1733646505104 (+19 ms)Region opened successfully at 1733646505104 2024-12-08T08:28:25,111 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd., pid=15, masterSystemTime=1733646505040 2024-12-08T08:28:25,119 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:28:25,119 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:28:25,124 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f82c14f86610fc5e8020f4821379cdbd, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:25,129 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f82c14f86610fc5e8020f4821379cdbd, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:25,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-08T08:28:25,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure f82c14f86610fc5e8020f4821379cdbd, server=422e41b75614,32911,1733646448831 in 259 msec 2024-12-08T08:28:25,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T08:28:25,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f82c14f86610fc5e8020f4821379cdbd, ASSIGN in 437 msec 2024-12-08T08:28:25,153 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:25,153 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646505153"}]},"ts":"1733646505153"} 2024-12-08T08:28:25,161 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-08T08:28:25,163 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:25,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 958 msec 2024-12-08T08:28:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T08:28:25,224 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-08T08:28:25,224 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-08T08:28:25,224 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:25,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-08T08:28:25,232 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:25,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-08T08:28:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:25,255 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:25,255 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-08T08:28:25,260 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-08T08:28:25,263 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-08T08:28:25,265 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-08T08:28:25,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=583559152127705113 2024-12-08T08:28:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646508264, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:25,270 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-08T08:28:25,273 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-08T08:28:25,274 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:25,276 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:25,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=583559152127705115 2024-12-08T08:28:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646508275, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:25,281 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:25,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=583559152127705116 2024-12-08T08:28:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646508281, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:25,287 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:25,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=583559152127705117 2024-12-08T08:28:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:46652 deadline: 1733646508286, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T08:28:25,292 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T08:28:25,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2141993544_22 at /127.0.0.1:39672 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 214) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7773 (was 7955) 2024-12-08T08:28:25,310 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=7773 2024-12-08T08:28:25,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:25,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-08T08:28:25,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:25,317 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:25,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-08T08:28:25,318 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-08T08:28:25,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741843_1019 (size=418) 2024-12-08T08:28:25,332 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4889d1457d6a210db6725bf12b2d7fd5, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:25,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741844_1020 (size=69) 2024-12-08T08:28:25,343 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:25,343 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 4889d1457d6a210db6725bf12b2d7fd5, disabling compactions & flushes 2024-12-08T08:28:25,343 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:28:25,343 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:28:25,343 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. after waiting 0 ms 2024-12-08T08:28:25,343 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:28:25,343 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:28:25,344 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4889d1457d6a210db6725bf12b2d7fd5: Waiting for close lock at 1733646505343Disabling compacts and flushes for region at 1733646505343Disabling writes for close at 1733646505343Writing region close event to WAL at 1733646505343Closed at 1733646505343 2024-12-08T08:28:25,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:25,346 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733646505346"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646505346"}]},"ts":"1733646505346"} 2024-12-08T08:28:25,350 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:25,352 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:25,352 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646505352"}]},"ts":"1733646505352"} 2024-12-08T08:28:25,356 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-08T08:28:25,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=4889d1457d6a210db6725bf12b2d7fd5, ASSIGN}] 2024-12-08T08:28:25,358 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=4889d1457d6a210db6725bf12b2d7fd5, ASSIGN 2024-12-08T08:28:25,360 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=4889d1457d6a210db6725bf12b2d7fd5, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:25,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4889d1457d6a210db6725bf12b2d7fd5, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:25,514 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=4889d1457d6a210db6725bf12b2d7fd5, ASSIGN because future has completed 2024-12-08T08:28:25,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4889d1457d6a210db6725bf12b2d7fd5, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:25,673 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:28:25,674 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 4889d1457d6a210db6725bf12b2d7fd5, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:25,674 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,674 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:25,675 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,675 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,677 INFO [StoreOpener-4889d1457d6a210db6725bf12b2d7fd5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,679 INFO [StoreOpener-4889d1457d6a210db6725bf12b2d7fd5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4889d1457d6a210db6725bf12b2d7fd5 columnFamilyName testFamily 2024-12-08T08:28:25,679 DEBUG [StoreOpener-4889d1457d6a210db6725bf12b2d7fd5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:25,680 INFO [StoreOpener-4889d1457d6a210db6725bf12b2d7fd5-1 {}] regionserver.HStore(327): Store=4889d1457d6a210db6725bf12b2d7fd5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:25,681 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,682 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,683 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,683 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,683 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,686 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,689 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:25,690 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 4889d1457d6a210db6725bf12b2d7fd5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61156746, jitterRate=-0.08869346976280212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:25,690 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:28:25,691 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 4889d1457d6a210db6725bf12b2d7fd5: Running coprocessor pre-open hook at 1733646505675Writing region info on filesystem at 1733646505675Initializing all the Stores at 1733646505676 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646505676Cleaning up temporary data from old regions at 1733646505684 (+8 ms)Running coprocessor post-open hooks at 1733646505690 (+6 ms)Region opened successfully at 1733646505691 (+1 ms) 2024-12-08T08:28:25,693 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., pid=18, masterSystemTime=1733646505668 2024-12-08T08:28:25,696 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:28:25,696 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:28:25,697 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=4889d1457d6a210db6725bf12b2d7fd5, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:25,701 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4889d1457d6a210db6725bf12b2d7fd5, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:25,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T08:28:25,710 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 4889d1457d6a210db6725bf12b2d7fd5, server=422e41b75614,32911,1733646448831 in 189 msec 2024-12-08T08:28:25,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-08T08:28:25,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=4889d1457d6a210db6725bf12b2d7fd5, ASSIGN in 354 msec 2024-12-08T08:28:25,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:25,718 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646505717"}]},"ts":"1733646505717"} 2024-12-08T08:28:25,721 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-08T08:28:25,723 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:25,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 410 msec 2024-12-08T08:28:25,910 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-08T08:28:25,911 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-08T08:28:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-08T08:28:26,333 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-08T08:28:26,334 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-08T08:28:26,334 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:26,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-08T08:28:26,340 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:26,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-08T08:28:26,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:26,362 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:26,363 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-08T08:28:26,367 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T08:28:26,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T08:28:27,055 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:28:27,374 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=422e41b75614:32911 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-08T08:28:27,375 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-08T08:28:27,375 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2 from cache 2024-12-08T08:28:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-08T08:28:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:28,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T08:28:28,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-08T08:28:28,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:28,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-08T08:28:28,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:29,374 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=422e41b75614:32911 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=978ms, rpcTimeout=971ms 2024-12-08T08:28:29,374 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=978ms, rpcTimeout=971ms 2024-12-08T08:28:29,374 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., hostname=422e41b75614,32911,1733646448831, seqNum=2 from cache 2024-12-08T08:28:29,384 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=7343 (was 7773) 2024-12-08T08:28:29,395 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=7342 2024-12-08T08:28:29,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-08T08:28:29,402 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:29,402 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:29,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-08T08:28:29,405 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:28:29,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741845_1021 (size=416) 2024-12-08T08:28:29,423 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3e6284cd8ab1289d786b0169bcbbc74d, NAME => 'testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:29,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741846_1022 (size=67) 2024-12-08T08:28:29,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:29,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 3e6284cd8ab1289d786b0169bcbbc74d, disabling compactions & flushes 2024-12-08T08:28:29,485 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:28:29,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:28:29,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. after waiting 0 ms 2024-12-08T08:28:29,485 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:28:29,485 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:28:29,486 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3e6284cd8ab1289d786b0169bcbbc74d: Waiting for close lock at 1733646509485Disabling compacts and flushes for region at 1733646509485Disabling writes for close at 1733646509485Writing region close event to WAL at 1733646509485Closed at 1733646509485 2024-12-08T08:28:29,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:29,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733646509490"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646509490"}]},"ts":"1733646509490"} 2024-12-08T08:28:29,498 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:29,501 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:29,501 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646509501"}]},"ts":"1733646509501"} 2024-12-08T08:28:29,510 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-08T08:28:29,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=3e6284cd8ab1289d786b0169bcbbc74d, ASSIGN}] 2024-12-08T08:28:29,512 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=3e6284cd8ab1289d786b0169bcbbc74d, ASSIGN 2024-12-08T08:28:29,514 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=3e6284cd8ab1289d786b0169bcbbc74d, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:29,665 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3e6284cd8ab1289d786b0169bcbbc74d, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:29,668 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=3e6284cd8ab1289d786b0169bcbbc74d, ASSIGN because future has completed 2024-12-08T08:28:29,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e6284cd8ab1289d786b0169bcbbc74d, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:29,830 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:28:29,831 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e6284cd8ab1289d786b0169bcbbc74d, NAME => 'testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:29,831 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,831 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:29,832 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,832 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,835 INFO [StoreOpener-3e6284cd8ab1289d786b0169bcbbc74d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,842 INFO [StoreOpener-3e6284cd8ab1289d786b0169bcbbc74d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e6284cd8ab1289d786b0169bcbbc74d columnFamilyName testFamily 2024-12-08T08:28:29,842 DEBUG [StoreOpener-3e6284cd8ab1289d786b0169bcbbc74d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:29,843 INFO [StoreOpener-3e6284cd8ab1289d786b0169bcbbc74d-1 {}] regionserver.HStore(327): Store=3e6284cd8ab1289d786b0169bcbbc74d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:29,843 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,849 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,850 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,851 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,852 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,858 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,866 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:29,867 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 3e6284cd8ab1289d786b0169bcbbc74d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68608254, jitterRate=0.022342652082443237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:29,867 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:28:29,867 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 3e6284cd8ab1289d786b0169bcbbc74d: Running coprocessor pre-open hook at 1733646509832Writing region info on filesystem at 1733646509832Initializing all the Stores at 1733646509834 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646509834Cleaning up temporary data from old regions at 1733646509852 (+18 ms)Running coprocessor post-open hooks at 1733646509867 (+15 ms)Region opened successfully at 1733646509867 2024-12-08T08:28:29,870 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d., pid=21, masterSystemTime=1733646509822 2024-12-08T08:28:29,878 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:28:29,878 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:28:29,878 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3e6284cd8ab1289d786b0169bcbbc74d, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:29,886 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e6284cd8ab1289d786b0169bcbbc74d, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-08T08:28:29,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-08T08:28:29,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e6284cd8ab1289d786b0169bcbbc74d, server=422e41b75614,32911,1733646448831 in 224 msec 2024-12-08T08:28:29,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T08:28:29,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=3e6284cd8ab1289d786b0169bcbbc74d, ASSIGN in 398 msec 2024-12-08T08:28:29,925 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:29,925 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646509925"}]},"ts":"1733646509925"} 2024-12-08T08:28:29,937 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-08T08:28:29,939 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:29,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 541 msec 2024-12-08T08:28:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T08:28:30,414 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-08T08:28:30,414 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-08T08:28:30,414 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:30,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-08T08:28:30,419 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:30,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-08T08:28:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:30,437 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:30,437 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-08T08:28:30,440 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T08:28:30,443 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T08:28:30,444 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T08:28:30,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-08T08:28:30,551 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:28:31,912 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-08T08:28:33,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32911 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705128' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T08:28:33,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46652 deadline: 1733646514445, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705128' does not exist 2024-12-08T08:28:33,449 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 422e41b75614,32911,1733646448831 for closing scanner id = 583559152127705128 for 3e6284cd8ab1289d786b0169bcbbc74d of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705128' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:28:33,461 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2141993544_22 at /127.0.0.1:39736 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 269) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6906 (was 7342) 2024-12-08T08:28:33,470 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6903 2024-12-08T08:28:33,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-08T08:28:33,478 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:33,478 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:33,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-08T08:28:33,481 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-08T08:28:33,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741847_1023 (size=411) 2024-12-08T08:28:33,503 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b70b7367dada62aecb3627ab6d09f42, NAME => 'testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741848_1024 (size=62) 2024-12-08T08:28:33,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:33,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 7b70b7367dada62aecb3627ab6d09f42, disabling compactions & flushes 2024-12-08T08:28:33,915 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:28:33,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:28:33,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. after waiting 0 ms 2024-12-08T08:28:33,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:28:33,915 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:28:33,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b70b7367dada62aecb3627ab6d09f42: Waiting for close lock at 1733646513915Disabling compacts and flushes for region at 1733646513915Disabling writes for close at 1733646513915Writing region close event to WAL at 1733646513915Closed at 1733646513915 2024-12-08T08:28:33,917 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:33,918 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733646513918"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646513918"}]},"ts":"1733646513918"} 2024-12-08T08:28:33,921 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:33,923 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:33,923 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646513923"}]},"ts":"1733646513923"} 2024-12-08T08:28:33,926 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-08T08:28:33,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7b70b7367dada62aecb3627ab6d09f42, ASSIGN}] 2024-12-08T08:28:33,929 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7b70b7367dada62aecb3627ab6d09f42, ASSIGN 2024-12-08T08:28:33,930 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7b70b7367dada62aecb3627ab6d09f42, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-08T08:28:34,081 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7b70b7367dada62aecb3627ab6d09f42, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:34,085 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7b70b7367dada62aecb3627ab6d09f42, ASSIGN because future has completed 2024-12-08T08:28:34,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b70b7367dada62aecb3627ab6d09f42, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:34,247 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:28:34,247 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b70b7367dada62aecb3627ab6d09f42, NAME => 'testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:34,247 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,248 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:34,248 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,248 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,251 INFO [StoreOpener-7b70b7367dada62aecb3627ab6d09f42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,257 INFO [StoreOpener-7b70b7367dada62aecb3627ab6d09f42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b70b7367dada62aecb3627ab6d09f42 columnFamilyName testFamily 2024-12-08T08:28:34,258 DEBUG [StoreOpener-7b70b7367dada62aecb3627ab6d09f42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:34,259 INFO [StoreOpener-7b70b7367dada62aecb3627ab6d09f42-1 {}] regionserver.HStore(327): Store=7b70b7367dada62aecb3627ab6d09f42/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:34,259 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,261 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,262 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,263 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,263 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,267 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,274 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:34,276 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 7b70b7367dada62aecb3627ab6d09f42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68755732, jitterRate=0.024540245532989502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:34,277 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:28:34,277 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 7b70b7367dada62aecb3627ab6d09f42: Running coprocessor pre-open hook at 1733646514248Writing region info on filesystem at 1733646514248Initializing all the Stores at 1733646514250 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646514250Cleaning up temporary data from old regions at 1733646514263 (+13 ms)Running coprocessor post-open hooks at 1733646514277 (+14 ms)Region opened successfully at 1733646514277 2024-12-08T08:28:34,279 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42., pid=24, masterSystemTime=1733646514240 2024-12-08T08:28:34,283 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:28:34,283 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:28:34,284 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7b70b7367dada62aecb3627ab6d09f42, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:34,298 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b70b7367dada62aecb3627ab6d09f42, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:34,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T08:28:34,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b70b7367dada62aecb3627ab6d09f42, server=422e41b75614,32911,1733646448831 in 223 msec 2024-12-08T08:28:34,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-08T08:28:34,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7b70b7367dada62aecb3627ab6d09f42, ASSIGN in 388 msec 2024-12-08T08:28:34,324 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:34,325 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646514325"}]},"ts":"1733646514325"} 2024-12-08T08:28:34,341 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-08T08:28:34,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:34,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 870 msec 2024-12-08T08:28:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-08T08:28:34,493 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-08T08:28:34,493 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-08T08:28:34,493 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:34,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-08T08:28:34,499 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:34,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-08T08:28:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:34,517 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:34,518 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-08T08:28:34,521 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T08:28:34,524 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T08:28:34,526 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T08:28:34,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-08T08:28:37,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32911 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705132' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T08:28:37,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46652 deadline: 1733646518538, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705132' does not exist 2024-12-08T08:28:37,540 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 422e41b75614,32911,1733646448831 for closing scanner id = 583559152127705132 for 7b70b7367dada62aecb3627ab6d09f42 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705132' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:28:37,548 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 223), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=327 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6524 (was 6903) 2024-12-08T08:28:37,562 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=327, ProcessCount=11, AvailableMemoryMB=6523 2024-12-08T08:28:37,565 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:37,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-08T08:28:37,570 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:37,570 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:37,570 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-08T08:28:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:28:37,572 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:37,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741849_1025 (size=419) 2024-12-08T08:28:37,633 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5fc91723157a98f7fc951956b88a7d8c, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741850_1026 (size=70) 2024-12-08T08:28:37,729 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:37,729 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 5fc91723157a98f7fc951956b88a7d8c, disabling compactions & flushes 2024-12-08T08:28:37,729 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:28:37,729 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:28:37,729 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. after waiting 0 ms 2024-12-08T08:28:37,729 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:28:37,729 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:28:37,729 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5fc91723157a98f7fc951956b88a7d8c: Waiting for close lock at 1733646517729Disabling compacts and flushes for region at 1733646517729Disabling writes for close at 1733646517729Writing region close event to WAL at 1733646517729Closed at 1733646517729 2024-12-08T08:28:37,736 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:37,737 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733646517737"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646517737"}]},"ts":"1733646517737"} 2024-12-08T08:28:37,755 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:37,761 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:37,761 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646517761"}]},"ts":"1733646517761"} 2024-12-08T08:28:37,775 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-08T08:28:37,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=5fc91723157a98f7fc951956b88a7d8c, ASSIGN}] 2024-12-08T08:28:37,783 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=5fc91723157a98f7fc951956b88a7d8c, ASSIGN 2024-12-08T08:28:37,785 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=5fc91723157a98f7fc951956b88a7d8c, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:37,914 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-08T08:28:37,937 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5fc91723157a98f7fc951956b88a7d8c, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:37,941 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=5fc91723157a98f7fc951956b88a7d8c, ASSIGN because future has completed 2024-12-08T08:28:37,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fc91723157a98f7fc951956b88a7d8c, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-08T08:28:38,103 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:28:38,103 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 5fc91723157a98f7fc951956b88a7d8c, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:38,104 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,104 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:38,104 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,104 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,113 INFO [StoreOpener-5fc91723157a98f7fc951956b88a7d8c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,130 INFO [StoreOpener-5fc91723157a98f7fc951956b88a7d8c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5fc91723157a98f7fc951956b88a7d8c columnFamilyName testFamily 2024-12-08T08:28:38,130 DEBUG [StoreOpener-5fc91723157a98f7fc951956b88a7d8c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:38,134 INFO [StoreOpener-5fc91723157a98f7fc951956b88a7d8c-1 {}] regionserver.HStore(327): Store=5fc91723157a98f7fc951956b88a7d8c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:38,134 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,149 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,154 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,169 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,190 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:38,191 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 5fc91723157a98f7fc951956b88a7d8c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62739586, jitterRate=-0.0651073157787323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:38,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:28:38,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 5fc91723157a98f7fc951956b88a7d8c: Running coprocessor pre-open hook at 1733646518104Writing region info on filesystem at 1733646518104Initializing all the Stores at 1733646518110 (+6 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646518111 (+1 ms)Cleaning up temporary data from old regions at 1733646518155 (+44 ms)Running coprocessor post-open hooks at 1733646518192 (+37 ms)Region opened successfully at 1733646518192 2024-12-08T08:28:38,194 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c., pid=27, masterSystemTime=1733646518096 2024-12-08T08:28:38,197 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:28:38,197 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:28:38,198 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=5fc91723157a98f7fc951956b88a7d8c, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:38,203 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fc91723157a98f7fc951956b88a7d8c, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:38,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-08T08:28:38,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 5fc91723157a98f7fc951956b88a7d8c, server=422e41b75614,32911,1733646448831 in 263 msec 2024-12-08T08:28:38,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T08:28:38,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=5fc91723157a98f7fc951956b88a7d8c, ASSIGN in 437 msec 2024-12-08T08:28:38,223 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:38,227 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646518227"}]},"ts":"1733646518227"} 2024-12-08T08:28:38,236 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-08T08:28:38,238 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:38,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 673 msec 2024-12-08T08:28:38,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-08T08:28:38,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:38,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-08T08:28:38,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:38,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-08T08:28:38,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:38,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T08:28:38,584 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-08T08:28:38,584 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-08T08:28:38,584 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:38,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-08T08:28:38,590 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:38,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-08T08:28:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:38,610 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:38,610 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-08T08:28:38,615 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T08:28:38,618 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-08T08:28:39,750 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:28:42,705 INFO [master/422e41b75614:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T08:28:42,705 INFO [master/422e41b75614:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T08:28:43,916 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-08T08:28:44,625 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c., hostname=422e41b75614,32911,1733646448831, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c., hostname=422e41b75614,32911,1733646448831, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=422e41b75614:32911 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-08T08:28:44,625 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c., hostname=422e41b75614,32911,1733646448831, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-08T08:28:44,625 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c., hostname=422e41b75614,32911,1733646448831, seqNum=2 from cache 2024-12-08T08:28:44,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=349 (was 327) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5194 (was 6523) 2024-12-08T08:28:44,646 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=349, ProcessCount=11, AvailableMemoryMB=5190 2024-12-08T08:28:44,649 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:44,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-08T08:28:44,653 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:44,654 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:44,654 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-08T08:28:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T08:28:44,656 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:44,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741851_1027 (size=407) 2024-12-08T08:28:45,073 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66847bb7174e461e7e27b0f9787d6387, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:45,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741852_1028 (size=58) 2024-12-08T08:28:45,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:45,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 66847bb7174e461e7e27b0f9787d6387, disabling compactions & flushes 2024-12-08T08:28:45,107 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:28:45,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:28:45,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. after waiting 0 ms 2024-12-08T08:28:45,108 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:28:45,108 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:28:45,108 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66847bb7174e461e7e27b0f9787d6387: Waiting for close lock at 1733646525107Disabling compacts and flushes for region at 1733646525107Disabling writes for close at 1733646525107Writing region close event to WAL at 1733646525108 (+1 ms)Closed at 1733646525108 2024-12-08T08:28:45,110 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:45,110 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733646525110"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646525110"}]},"ts":"1733646525110"} 2024-12-08T08:28:45,115 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:45,117 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:45,117 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646525117"}]},"ts":"1733646525117"} 2024-12-08T08:28:45,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-08T08:28:45,120 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-08T08:28:45,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=66847bb7174e461e7e27b0f9787d6387, ASSIGN}] 2024-12-08T08:28:45,122 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=66847bb7174e461e7e27b0f9787d6387, ASSIGN 2024-12-08T08:28:45,124 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=66847bb7174e461e7e27b0f9787d6387, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:45,275 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=66847bb7174e461e7e27b0f9787d6387, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:45,280 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=66847bb7174e461e7e27b0f9787d6387, ASSIGN because future has completed 2024-12-08T08:28:45,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66847bb7174e461e7e27b0f9787d6387, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:45,440 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:28:45,440 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 66847bb7174e461e7e27b0f9787d6387, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:45,441 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,441 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:45,441 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,441 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,443 INFO [StoreOpener-66847bb7174e461e7e27b0f9787d6387-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,447 INFO [StoreOpener-66847bb7174e461e7e27b0f9787d6387-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66847bb7174e461e7e27b0f9787d6387 columnFamilyName testFamily 2024-12-08T08:28:45,447 DEBUG [StoreOpener-66847bb7174e461e7e27b0f9787d6387-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:45,448 INFO [StoreOpener-66847bb7174e461e7e27b0f9787d6387-1 {}] regionserver.HStore(327): Store=66847bb7174e461e7e27b0f9787d6387/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:45,448 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,449 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,453 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,454 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,454 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,457 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,462 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:45,463 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 66847bb7174e461e7e27b0f9787d6387; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72148318, jitterRate=0.07509371638298035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:45,463 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:28:45,464 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 66847bb7174e461e7e27b0f9787d6387: Running coprocessor pre-open hook at 1733646525441Writing region info on filesystem at 1733646525441Initializing all the Stores at 1733646525443 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646525443Cleaning up temporary data from old regions at 1733646525454 (+11 ms)Running coprocessor post-open hooks at 1733646525463 (+9 ms)Region opened successfully at 1733646525464 (+1 ms) 2024-12-08T08:28:45,465 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387., pid=30, masterSystemTime=1733646525434 2024-12-08T08:28:45,468 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:28:45,469 INFO [RS_OPEN_PRIORITY_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:28:45,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=66847bb7174e461e7e27b0f9787d6387, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:45,478 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66847bb7174e461e7e27b0f9787d6387, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:45,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T08:28:45,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 66847bb7174e461e7e27b0f9787d6387, server=422e41b75614,32911,1733646448831 in 200 msec 2024-12-08T08:28:45,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-08T08:28:45,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=66847bb7174e461e7e27b0f9787d6387, ASSIGN in 365 msec 2024-12-08T08:28:45,492 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:45,493 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646525493"}]},"ts":"1733646525493"} 2024-12-08T08:28:45,498 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-08T08:28:45,503 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:45,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 853 msec 2024-12-08T08:28:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T08:28:45,664 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-08T08:28:45,664 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-08T08:28:45,664 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:45,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-08T08:28:45,671 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:28:45,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-08T08:28:45,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:28:45,694 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:28:45,694 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-08T08:28:45,700 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T08:28:45,708 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T08:28:45,709 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T08:28:45,709 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-08T08:28:48,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-08T08:28:48,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-08T08:28:49,918 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-08T08:28:50,972 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:28:51,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32911 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705140' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T08:28:51,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:46652 deadline: 1733646537715, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705140' does not exist 2024-12-08T08:28:51,719 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 422e41b75614,32911,1733646448831 for closing scanner id = 583559152127705140 for 66847bb7174e461e7e27b0f9787d6387 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '583559152127705140' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T08:28:51,724 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 349) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3644 (was 5190) 2024-12-08T08:28:51,732 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=3644 2024-12-08T08:28:51,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T08:28:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-08T08:28:51,738 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T08:28:51,738 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:51,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-08T08:28:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T08:28:51,739 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T08:28:51,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741853_1029 (size=423) 2024-12-08T08:28:52,149 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe49dfd0c5ac08c96231aaa9c6b37364, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4 2024-12-08T08:28:52,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741854_1030 (size=74) 2024-12-08T08:28:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-08T08:28:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T08:28:52,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:52,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing fe49dfd0c5ac08c96231aaa9c6b37364, disabling compactions & flushes 2024-12-08T08:28:52,916 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:28:52,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:28:52,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. after waiting 0 ms 2024-12-08T08:28:52,917 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:28:52,917 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:28:52,917 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe49dfd0c5ac08c96231aaa9c6b37364: Waiting for close lock at 1733646532916Disabling compacts and flushes for region at 1733646532916Disabling writes for close at 1733646532916Writing region close event to WAL at 1733646532917 (+1 ms)Closed at 1733646532917 2024-12-08T08:28:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T08:28:57,055 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T08:28:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T08:28:58,675 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1025680706_22 at /127.0.0.1:43948 [Receiving block BP-242960261-172.17.0.3-1733646444019:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 5757ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/data/data2/, blockId=1073741830, seqno=279 2024-12-08T08:28:58,683 INFO [AsyncFSWAL-0-hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData-prefix:422e41b75614,33175,1733646448021 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5765 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37723,DS-cb92d21b-4ff2-45cb-a181-d74bbe865e3c,DISK]] 2024-12-08T08:28:58,683 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T08:28:58,683 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733646538683"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733646538683"}]},"ts":"1733646538683"} 2024-12-08T08:28:58,688 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T08:28:58,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T08:28:58,690 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646538690"}]},"ts":"1733646538690"} 2024-12-08T08:28:58,693 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-08T08:28:58,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fe49dfd0c5ac08c96231aaa9c6b37364, ASSIGN}] 2024-12-08T08:28:58,696 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fe49dfd0c5ac08c96231aaa9c6b37364, ASSIGN 2024-12-08T08:28:58,697 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fe49dfd0c5ac08c96231aaa9c6b37364, ASSIGN; state=OFFLINE, location=422e41b75614,32911,1733646448831; forceNewPlan=false, retain=false 2024-12-08T08:28:58,848 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=fe49dfd0c5ac08c96231aaa9c6b37364, regionState=OPENING, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:58,852 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fe49dfd0c5ac08c96231aaa9c6b37364, ASSIGN because future has completed 2024-12-08T08:28:58,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe49dfd0c5ac08c96231aaa9c6b37364, server=422e41b75614,32911,1733646448831}] 2024-12-08T08:28:59,018 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:28:59,018 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => fe49dfd0c5ac08c96231aaa9c6b37364, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364.', STARTKEY => '', ENDKEY => ''} 2024-12-08T08:28:59,019 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,019 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T08:28:59,019 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,019 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,029 INFO [StoreOpener-fe49dfd0c5ac08c96231aaa9c6b37364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,031 INFO [StoreOpener-fe49dfd0c5ac08c96231aaa9c6b37364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe49dfd0c5ac08c96231aaa9c6b37364 columnFamilyName testFamily 2024-12-08T08:28:59,031 DEBUG [StoreOpener-fe49dfd0c5ac08c96231aaa9c6b37364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T08:28:59,032 INFO [StoreOpener-fe49dfd0c5ac08c96231aaa9c6b37364-1 {}] regionserver.HStore(327): Store=fe49dfd0c5ac08c96231aaa9c6b37364/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T08:28:59,033 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,036 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,037 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,037 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,038 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,040 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,045 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T08:28:59,046 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened fe49dfd0c5ac08c96231aaa9c6b37364; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74087789, jitterRate=0.10399408638477325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T08:28:59,046 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:28:59,047 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for fe49dfd0c5ac08c96231aaa9c6b37364: Running coprocessor pre-open hook at 1733646539019Writing region info on filesystem at 1733646539019Initializing all the Stores at 1733646539022 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733646539022Cleaning up temporary data from old regions at 1733646539038 (+16 ms)Running coprocessor post-open hooks at 1733646539046 (+8 ms)Region opened successfully at 1733646539047 (+1 ms) 2024-12-08T08:28:59,048 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., pid=33, masterSystemTime=1733646539006 2024-12-08T08:28:59,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=fe49dfd0c5ac08c96231aaa9c6b37364, regionState=OPEN, openSeqNum=2, regionLocation=422e41b75614,32911,1733646448831 2024-12-08T08:28:59,053 DEBUG [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:28:59,053 INFO [RS_OPEN_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:28:59,060 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe49dfd0c5ac08c96231aaa9c6b37364, server=422e41b75614,32911,1733646448831 because future has completed 2024-12-08T08:28:59,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T08:28:59,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure fe49dfd0c5ac08c96231aaa9c6b37364, server=422e41b75614,32911,1733646448831 in 215 msec 2024-12-08T08:28:59,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-08T08:28:59,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=fe49dfd0c5ac08c96231aaa9c6b37364, ASSIGN in 379 msec 2024-12-08T08:28:59,080 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T08:28:59,080 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733646539080"}]},"ts":"1733646539080"} 2024-12-08T08:28:59,084 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-08T08:28:59,087 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T08:28:59,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 7.3520 sec 2024-12-08T08:29:01,920 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-08T08:29:02,547 DEBUG [master/422e41b75614:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-08T08:29:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33175 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T08:29:02,823 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-08T08:29:02,823 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-08T08:29:02,823 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:29:02,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-08T08:29:02,830 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T08:29:02,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-08T08:29:02,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:29:02,850 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T08:29:02,850 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-08T08:29:02,853 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T08:29:02,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T08:29:03,864 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=422e41b75614:32911 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-12-08T08:29:03,864 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-12-08T08:29:03,864 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2 from cache 2024-12-08T08:29:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-08T08:29:04,542 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T08:29:04,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2] 2024-12-08T08:29:04,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32911 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T08:29:05,854 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=422e41b75614:32911 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=969ms, rpcTimeout=968ms 2024-12-08T08:29:05,854 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=969ms, rpcTimeout=968ms 2024-12-08T08:29:05,854 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., hostname=422e41b75614,32911,1733646448831, seqNum=2 from cache 2024-12-08T08:29:05,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=607 (was 488) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3088 (was 3644) 2024-12-08T08:29:05,862 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:29:05,862 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:29:05,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:29:05,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:29:05,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:29:05,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T08:29:05,868 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:29:05,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:29:05,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:29:05,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:29:05,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T08:29:05,868 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T08:29:05,868 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=331909808, stopped=false 2024-12-08T08:29:05,869 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=422e41b75614,33175,1733646448021 2024-12-08T08:29:06,387 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-08T08:29:08,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-08T08:29:08,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T08:29:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T08:29:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T08:29:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:29:10,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:29:10,080 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T08:29:10,081 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T08:29:10,081 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:29:10,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:29:10,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:29:10,081 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '422e41b75614,32911,1733646448831' ***** 2024-12-08T08:29:10,082 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T08:29:10,082 INFO [RS:0;422e41b75614:32911 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T08:29:10,082 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T08:29:10,082 INFO [RS:0;422e41b75614:32911 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T08:29:10,082 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T08:29:10,082 INFO [RS:0;422e41b75614:32911 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T08:29:10,083 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 5fc91723157a98f7fc951956b88a7d8c 2024-12-08T08:29:10,083 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 66847bb7174e461e7e27b0f9787d6387 2024-12-08T08:29:10,083 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b70b7367dada62aecb3627ab6d09f42 2024-12-08T08:29:10,083 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 15b9440e89d0596da15af951d1128587 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for f82c14f86610fc5e8020f4821379cdbd 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for efb3a6942cdf755b7102cf2ed271587b 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 07499732554247c1fe88f68b036dfcc1 2024-12-08T08:29:10,084 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5fc91723157a98f7fc951956b88a7d8c, disabling compactions & flushes 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:29:10,084 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:29:10,084 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(959): stopping server 422e41b75614,32911,1733646448831 2024-12-08T08:29:10,084 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. after waiting 0 ms 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T08:29:10,084 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:29:10,084 INFO [RS:0;422e41b75614:32911 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;422e41b75614:32911. 2024-12-08T08:29:10,084 DEBUG [RS:0;422e41b75614:32911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T08:29:10,084 DEBUG [RS:0;422e41b75614:32911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:29:10,085 INFO [RS:0;422e41b75614:32911 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T08:29:10,085 INFO [RS:0;422e41b75614:32911 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T08:29:10,085 INFO [RS:0;422e41b75614:32911 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T08:29:10,085 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T08:29:10,085 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-08T08:29:10,086 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T08:29:10,086 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T08:29:10,086 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T08:29:10,086 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T08:29:10,086 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T08:29:10,086 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 1588230740, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, 5fc91723157a98f7fc951956b88a7d8c, 66847bb7174e461e7e27b0f9787d6387, 7b70b7367dada62aecb3627ab6d09f42, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:10,089 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5fc91723157a98f7fc951956b88a7d8c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:10,089 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-08T08:29:10,160 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c/.tmp/testFamily/46e3ac75c95f45c2a4ea4cacf828199c is 61, key is row-0/testFamily:testQualifier/1733646518596/Put/seqid=0 2024-12-08T08:29:10,166 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/info/8eed01a375154640979c68e175ef6426 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1./info:regioninfo/1733646484035/Put/seqid=0 2024-12-08T08:29:10,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741855_1031 (size=5049) 2024-12-08T08:29:10,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741856_1032 (size=20564) 2024-12-08T08:29:10,186 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/info/8eed01a375154640979c68e175ef6426 2024-12-08T08:29:10,275 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/ns/e253374ca6364da0b41c597fbab12f49 is 43, key is default/ns:d/1733646482489/Put/seqid=0 2024-12-08T08:29:10,289 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 1588230740, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, 5fc91723157a98f7fc951956b88a7d8c, 66847bb7174e461e7e27b0f9787d6387, 7b70b7367dada62aecb3627ab6d09f42, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:10,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741857_1033 (size=5153) 2024-12-08T08:29:10,489 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 1588230740, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, 5fc91723157a98f7fc951956b88a7d8c, 66847bb7174e461e7e27b0f9787d6387, 7b70b7367dada62aecb3627ab6d09f42, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:10,582 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c/.tmp/testFamily/46e3ac75c95f45c2a4ea4cacf828199c 2024-12-08T08:29:10,600 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c/.tmp/testFamily/46e3ac75c95f45c2a4ea4cacf828199c as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c/testFamily/46e3ac75c95f45c2a4ea4cacf828199c 2024-12-08T08:29:10,616 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c/testFamily/46e3ac75c95f45c2a4ea4cacf828199c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:10,629 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5fc91723157a98f7fc951956b88a7d8c in 537ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:10,666 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/5fc91723157a98f7fc951956b88a7d8c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:10,675 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:29:10,675 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5fc91723157a98f7fc951956b88a7d8c: Waiting for close lock at 1733646550083Running coprocessor pre-close hooks at 1733646550084 (+1 ms)Disabling compacts and flushes for region at 1733646550084Disabling writes for close at 1733646550084Obtaining lock to block concurrent updates at 1733646550089 (+5 ms)Preparing flush snapshotting stores in 5fc91723157a98f7fc951956b88a7d8c at 1733646550089Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646550096 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. at 1733646550098 (+2 ms)Flushing 5fc91723157a98f7fc951956b88a7d8c/testFamily: creating writer at 1733646550099 (+1 ms)Flushing 5fc91723157a98f7fc951956b88a7d8c/testFamily: appending metadata at 1733646550153 (+54 ms)Flushing 5fc91723157a98f7fc951956b88a7d8c/testFamily: closing flushed file at 1733646550156 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173395d1: reopening flushed file at 1733646550597 (+441 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5fc91723157a98f7fc951956b88a7d8c in 537ms, sequenceid=8, compaction requested=false at 1733646550629 (+32 ms)Writing region close event to WAL at 1733646550637 (+8 ms)Running coprocessor post-close hooks at 1733646550672 (+35 ms)Closed at 1733646550675 (+3 ms) 2024-12-08T08:29:10,676 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733646517565.5fc91723157a98f7fc951956b88a7d8c. 2024-12-08T08:29:10,676 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66847bb7174e461e7e27b0f9787d6387, disabling compactions & flushes 2024-12-08T08:29:10,676 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:29:10,676 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:29:10,676 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. after waiting 0 ms 2024-12-08T08:29:10,676 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:29:10,677 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 66847bb7174e461e7e27b0f9787d6387 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:10,693 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 1588230740, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, 66847bb7174e461e7e27b0f9787d6387, 7b70b7367dada62aecb3627ab6d09f42, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:10,711 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/ns/e253374ca6364da0b41c597fbab12f49 2024-12-08T08:29:10,747 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387/.tmp/testFamily/d80c7d76cb264461aecf37d5620a17ad is 61, key is row-0/testFamily:testQualifier/1733646525678/Put/seqid=0 2024-12-08T08:29:10,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741858_1034 (size=5049) 2024-12-08T08:29:10,813 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/table/9053693b994040c59e79a1adc88b80b4 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733646484050/Put/seqid=0 2024-12-08T08:29:10,814 INFO [regionserver/422e41b75614:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T08:29:10,815 INFO [regionserver/422e41b75614:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T08:29:10,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741859_1035 (size=6699) 2024-12-08T08:29:10,824 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/table/9053693b994040c59e79a1adc88b80b4 2024-12-08T08:29:10,853 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/info/8eed01a375154640979c68e175ef6426 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/info/8eed01a375154640979c68e175ef6426 2024-12-08T08:29:10,896 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/info/8eed01a375154640979c68e175ef6426, entries=100, sequenceid=56, filesize=20.1 K 2024-12-08T08:29:10,897 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 1588230740, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, 66847bb7174e461e7e27b0f9787d6387, 7b70b7367dada62aecb3627ab6d09f42, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:10,898 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/ns/e253374ca6364da0b41c597fbab12f49 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/ns/e253374ca6364da0b41c597fbab12f49 2024-12-08T08:29:10,919 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/ns/e253374ca6364da0b41c597fbab12f49, entries=2, sequenceid=56, filesize=5.0 K 2024-12-08T08:29:10,926 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/.tmp/table/9053693b994040c59e79a1adc88b80b4 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/table/9053693b994040c59e79a1adc88b80b4 2024-12-08T08:29:10,957 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/table/9053693b994040c59e79a1adc88b80b4, entries=20, sequenceid=56, filesize=6.5 K 2024-12-08T08:29:10,972 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 885ms, sequenceid=56, compaction requested=false 2024-12-08T08:29:11,024 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-08T08:29:11,029 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T08:29:11,029 INFO [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T08:29:11,033 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733646550086Running coprocessor pre-close hooks at 1733646550086Disabling compacts and flushes for region at 1733646550086Disabling writes for close at 1733646550086Obtaining lock to block concurrent updates at 1733646550089 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733646550089Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733646550096 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733646550098 (+2 ms)Flushing 1588230740/info: creating writer at 1733646550099 (+1 ms)Flushing 1588230740/info: appending metadata at 1733646550161 (+62 ms)Flushing 1588230740/info: closing flushed file at 1733646550161Flushing 1588230740/ns: creating writer at 1733646550247 (+86 ms)Flushing 1588230740/ns: appending metadata at 1733646550272 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1733646550273 (+1 ms)Flushing 1588230740/table: creating writer at 1733646550759 (+486 ms)Flushing 1588230740/table: appending metadata at 1733646550812 (+53 ms)Flushing 1588230740/table: closing flushed file at 1733646550812Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b9cb413: reopening flushed file at 1733646550841 (+29 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bbccd5e: reopening flushed file at 1733646550896 (+55 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73846e40: reopening flushed file at 1733646550919 (+23 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 885ms, sequenceid=56, compaction requested=false at 1733646550972 (+53 ms)Writing region close event to WAL at 1733646550989 (+17 ms)Running coprocessor post-close hooks at 1733646551028 (+39 ms)Closed at 1733646551029 (+1 ms) 2024-12-08T08:29:11,033 DEBUG [RS_CLOSE_META-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T08:29:11,097 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-08T08:29:11,097 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1325): Online Regions={66847bb7174e461e7e27b0f9787d6387=hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387., 7b70b7367dada62aecb3627ab6d09f42=testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42., 15b9440e89d0596da15af951d1128587=hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., f82c14f86610fc5e8020f4821379cdbd=testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd., efb3a6942cdf755b7102cf2ed271587b=hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b., fe49dfd0c5ac08c96231aaa9c6b37364=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., 4889d1457d6a210db6725bf12b2d7fd5=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., 07499732554247c1fe88f68b036dfcc1=testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1., 3e6284cd8ab1289d786b0169bcbbc74d=testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.} 2024-12-08T08:29:11,097 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, 66847bb7174e461e7e27b0f9787d6387, 7b70b7367dada62aecb3627ab6d09f42, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:11,201 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387/.tmp/testFamily/d80c7d76cb264461aecf37d5620a17ad 2024-12-08T08:29:11,228 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387/.tmp/testFamily/d80c7d76cb264461aecf37d5620a17ad as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387/testFamily/d80c7d76cb264461aecf37d5620a17ad 2024-12-08T08:29:11,242 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387/testFamily/d80c7d76cb264461aecf37d5620a17ad, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:11,244 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 66847bb7174e461e7e27b0f9787d6387 in 567ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:11,255 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNext/66847bb7174e461e7e27b0f9787d6387/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:11,257 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:29:11,257 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66847bb7174e461e7e27b0f9787d6387: Waiting for close lock at 1733646550676Running coprocessor pre-close hooks at 1733646550676Disabling compacts and flushes for region at 1733646550676Disabling writes for close at 1733646550676Obtaining lock to block concurrent updates at 1733646550677 (+1 ms)Preparing flush snapshotting stores in 66847bb7174e461e7e27b0f9787d6387 at 1733646550677Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646550677Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. at 1733646550689 (+12 ms)Flushing 66847bb7174e461e7e27b0f9787d6387/testFamily: creating writer at 1733646550689Flushing 66847bb7174e461e7e27b0f9787d6387/testFamily: appending metadata at 1733646550747 (+58 ms)Flushing 66847bb7174e461e7e27b0f9787d6387/testFamily: closing flushed file at 1733646550747Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@acf20bc: reopening flushed file at 1733646551226 (+479 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 66847bb7174e461e7e27b0f9787d6387 in 567ms, sequenceid=8, compaction requested=false at 1733646551244 (+18 ms)Writing region close event to WAL at 1733646551245 (+1 ms)Running coprocessor post-close hooks at 1733646551257 (+12 ms)Closed at 1733646551257 2024-12-08T08:29:11,257 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733646524649.66847bb7174e461e7e27b0f9787d6387. 2024-12-08T08:29:11,258 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b70b7367dada62aecb3627ab6d09f42, disabling compactions & flushes 2024-12-08T08:29:11,258 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:29:11,258 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:29:11,258 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. after waiting 0 ms 2024-12-08T08:29:11,258 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:29:11,258 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7b70b7367dada62aecb3627ab6d09f42 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:11,282 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42/.tmp/testFamily/9c1fbc6076ae4f5c863facf81c165e22 is 61, key is row-0/testFamily:testQualifier/1733646514505/Put/seqid=0 2024-12-08T08:29:11,297 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, 7b70b7367dada62aecb3627ab6d09f42, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:11,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741860_1036 (size=5049) 2024-12-08T08:29:11,310 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42/.tmp/testFamily/9c1fbc6076ae4f5c863facf81c165e22 2024-12-08T08:29:11,341 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42/.tmp/testFamily/9c1fbc6076ae4f5c863facf81c165e22 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42/testFamily/9c1fbc6076ae4f5c863facf81c165e22 2024-12-08T08:29:11,365 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42/testFamily/9c1fbc6076ae4f5c863facf81c165e22, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:11,366 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7b70b7367dada62aecb3627ab6d09f42 in 108ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:11,378 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNext/7b70b7367dada62aecb3627ab6d09f42/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:11,379 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:29:11,379 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b70b7367dada62aecb3627ab6d09f42: Waiting for close lock at 1733646551258Running coprocessor pre-close hooks at 1733646551258Disabling compacts and flushes for region at 1733646551258Disabling writes for close at 1733646551258Obtaining lock to block concurrent updates at 1733646551258Preparing flush snapshotting stores in 7b70b7367dada62aecb3627ab6d09f42 at 1733646551258Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646551258Flushing stores of testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. at 1733646551259 (+1 ms)Flushing 7b70b7367dada62aecb3627ab6d09f42/testFamily: creating writer at 1733646551259Flushing 7b70b7367dada62aecb3627ab6d09f42/testFamily: appending metadata at 1733646551281 (+22 ms)Flushing 7b70b7367dada62aecb3627ab6d09f42/testFamily: closing flushed file at 1733646551281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5641bbb5: reopening flushed file at 1733646551333 (+52 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7b70b7367dada62aecb3627ab6d09f42 in 108ms, sequenceid=8, compaction requested=false at 1733646551366 (+33 ms)Writing region close event to WAL at 1733646551367 (+1 ms)Running coprocessor post-close hooks at 1733646551378 (+11 ms)Closed at 1733646551379 (+1 ms) 2024-12-08T08:29:11,379 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733646513472.7b70b7367dada62aecb3627ab6d09f42. 2024-12-08T08:29:11,379 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 15b9440e89d0596da15af951d1128587, disabling compactions & flushes 2024-12-08T08:29:11,380 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:29:11,380 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:29:11,380 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. after waiting 0 ms 2024-12-08T08:29:11,380 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:29:11,380 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 15b9440e89d0596da15af951d1128587 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:11,421 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587/.tmp/testFamily/c4974067ac8943c6b37d669ad4a1ebf2 is 61, key is row-0/testFamily:testQualifier/1733646489047/Put/seqid=0 2024-12-08T08:29:11,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741861_1037 (size=5049) 2024-12-08T08:29:11,455 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587/.tmp/testFamily/c4974067ac8943c6b37d669ad4a1ebf2 2024-12-08T08:29:11,489 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587/.tmp/testFamily/c4974067ac8943c6b37d669ad4a1ebf2 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587/testFamily/c4974067ac8943c6b37d669ad4a1ebf2 2024-12-08T08:29:11,498 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 15b9440e89d0596da15af951d1128587, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:11,501 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587/testFamily/c4974067ac8943c6b37d669ad4a1ebf2, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:11,505 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 15b9440e89d0596da15af951d1128587 in 125ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:11,556 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnOpenScanner/15b9440e89d0596da15af951d1128587/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:11,559 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:29:11,559 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 15b9440e89d0596da15af951d1128587: Waiting for close lock at 1733646551379Running coprocessor pre-close hooks at 1733646551379Disabling compacts and flushes for region at 1733646551379Disabling writes for close at 1733646551380 (+1 ms)Obtaining lock to block concurrent updates at 1733646551380Preparing flush snapshotting stores in 15b9440e89d0596da15af951d1128587 at 1733646551380Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646551380Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. at 1733646551381 (+1 ms)Flushing 15b9440e89d0596da15af951d1128587/testFamily: creating writer at 1733646551381Flushing 15b9440e89d0596da15af951d1128587/testFamily: appending metadata at 1733646551420 (+39 ms)Flushing 15b9440e89d0596da15af951d1128587/testFamily: closing flushed file at 1733646551420Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4aed8d34: reopening flushed file at 1733646551487 (+67 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 15b9440e89d0596da15af951d1128587 in 125ms, sequenceid=8, compaction requested=false at 1733646551505 (+18 ms)Writing region close event to WAL at 1733646551525 (+20 ms)Running coprocessor post-close hooks at 1733646551559 (+34 ms)Closed at 1733646551559 2024-12-08T08:29:11,559 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733646485984.15b9440e89d0596da15af951d1128587. 2024-12-08T08:29:11,559 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f82c14f86610fc5e8020f4821379cdbd, disabling compactions & flushes 2024-12-08T08:29:11,559 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:29:11,559 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:29:11,560 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. after waiting 0 ms 2024-12-08T08:29:11,560 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:29:11,560 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f82c14f86610fc5e8020f4821379cdbd 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:11,598 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd/.tmp/testFamily/3fec14a3d7f84fb8baa34972231cc059 is 61, key is row-0/testFamily:testQualifier/1733646505239/Put/seqid=0 2024-12-08T08:29:11,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741862_1038 (size=5049) 2024-12-08T08:29:11,698 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:11,898 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, efb3a6942cdf755b7102cf2ed271587b, f82c14f86610fc5e8020f4821379cdbd, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:12,036 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd/.tmp/testFamily/3fec14a3d7f84fb8baa34972231cc059 2024-12-08T08:29:12,050 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd/.tmp/testFamily/3fec14a3d7f84fb8baa34972231cc059 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd/testFamily/3fec14a3d7f84fb8baa34972231cc059 2024-12-08T08:29:12,063 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd/testFamily/3fec14a3d7f84fb8baa34972231cc059, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:12,072 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f82c14f86610fc5e8020f4821379cdbd in 512ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:12,098 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextException/f82c14f86610fc5e8020f4821379cdbd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:12,099 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:29:12,100 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f82c14f86610fc5e8020f4821379cdbd: Waiting for close lock at 1733646551559Running coprocessor pre-close hooks at 1733646551559Disabling compacts and flushes for region at 1733646551559Disabling writes for close at 1733646551560 (+1 ms)Obtaining lock to block concurrent updates at 1733646551560Preparing flush snapshotting stores in f82c14f86610fc5e8020f4821379cdbd at 1733646551560Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646551560Flushing stores of testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. at 1733646551561 (+1 ms)Flushing f82c14f86610fc5e8020f4821379cdbd/testFamily: creating writer at 1733646551561Flushing f82c14f86610fc5e8020f4821379cdbd/testFamily: appending metadata at 1733646551597 (+36 ms)Flushing f82c14f86610fc5e8020f4821379cdbd/testFamily: closing flushed file at 1733646551597Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49c28963: reopening flushed file at 1733646552048 (+451 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f82c14f86610fc5e8020f4821379cdbd in 512ms, sequenceid=8, compaction requested=false at 1733646552072 (+24 ms)Writing region close event to WAL at 1733646552085 (+13 ms)Running coprocessor post-close hooks at 1733646552099 (+14 ms)Closed at 1733646552099 2024-12-08T08:29:12,100 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733646504205.f82c14f86610fc5e8020f4821379cdbd. 2024-12-08T08:29:12,100 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing efb3a6942cdf755b7102cf2ed271587b, disabling compactions & flushes 2024-12-08T08:29:12,100 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:29:12,100 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:29:12,100 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. after waiting 0 ms 2024-12-08T08:29:12,100 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-12-08T08:29:12,100 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:29:12,100 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1325): Online Regions={efb3a6942cdf755b7102cf2ed271587b=hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b., fe49dfd0c5ac08c96231aaa9c6b37364=testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., 4889d1457d6a210db6725bf12b2d7fd5=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., 07499732554247c1fe88f68b036dfcc1=testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1., 3e6284cd8ab1289d786b0169bcbbc74d=testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.} 2024-12-08T08:29:12,101 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing efb3a6942cdf755b7102cf2ed271587b 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:12,101 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, efb3a6942cdf755b7102cf2ed271587b, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:12,127 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b/.tmp/testFamily/b5f8c255c0fc4bf0bef724029d3c1868 is 61, key is row-0/testFamily:testQualifier/1733646498153/Put/seqid=0 2024-12-08T08:29:12,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741863_1039 (size=5049) 2024-12-08T08:29:12,301 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, efb3a6942cdf755b7102cf2ed271587b, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:12,501 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5, efb3a6942cdf755b7102cf2ed271587b, fe49dfd0c5ac08c96231aaa9c6b37364 2024-12-08T08:29:12,544 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b/.tmp/testFamily/b5f8c255c0fc4bf0bef724029d3c1868 2024-12-08T08:29:12,559 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b/.tmp/testFamily/b5f8c255c0fc4bf0bef724029d3c1868 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b/testFamily/b5f8c255c0fc4bf0bef724029d3c1868 2024-12-08T08:29:12,577 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b/testFamily/b5f8c255c0fc4bf0bef724029d3c1868, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:12,578 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for efb3a6942cdf755b7102cf2ed271587b in 478ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:12,591 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/hbase/testMetaScanTimeoutOnNextAsync/efb3a6942cdf755b7102cf2ed271587b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:12,595 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:29:12,595 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for efb3a6942cdf755b7102cf2ed271587b: Waiting for close lock at 1733646552100Running coprocessor pre-close hooks at 1733646552100Disabling compacts and flushes for region at 1733646552100Disabling writes for close at 1733646552100Obtaining lock to block concurrent updates at 1733646552101 (+1 ms)Preparing flush snapshotting stores in efb3a6942cdf755b7102cf2ed271587b at 1733646552101Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646552101Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. at 1733646552104 (+3 ms)Flushing efb3a6942cdf755b7102cf2ed271587b/testFamily: creating writer at 1733646552104Flushing efb3a6942cdf755b7102cf2ed271587b/testFamily: appending metadata at 1733646552127 (+23 ms)Flushing efb3a6942cdf755b7102cf2ed271587b/testFamily: closing flushed file at 1733646552127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@422053ee: reopening flushed file at 1733646552557 (+430 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for efb3a6942cdf755b7102cf2ed271587b in 478ms, sequenceid=8, compaction requested=false at 1733646552578 (+21 ms)Writing region close event to WAL at 1733646552582 (+4 ms)Running coprocessor post-close hooks at 1733646552595 (+13 ms)Closed at 1733646552595 2024-12-08T08:29:12,596 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733646495095.efb3a6942cdf755b7102cf2ed271587b. 2024-12-08T08:29:12,596 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fe49dfd0c5ac08c96231aaa9c6b37364, disabling compactions & flushes 2024-12-08T08:29:12,596 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:29:12,596 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:29:12,596 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. after waiting 0 ms 2024-12-08T08:29:12,596 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:29:12,597 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fe49dfd0c5ac08c96231aaa9c6b37364 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:12,629 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364/.tmp/testFamily/848a7772c59d474d8344f049156ef744 is 61, key is row-0/testFamily:testQualifier/1733646542837/Put/seqid=0 2024-12-08T08:29:12,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741864_1040 (size=5049) 2024-12-08T08:29:12,643 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364/.tmp/testFamily/848a7772c59d474d8344f049156ef744 2024-12-08T08:29:12,658 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364/.tmp/testFamily/848a7772c59d474d8344f049156ef744 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364/testFamily/848a7772c59d474d8344f049156ef744 2024-12-08T08:29:12,670 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364/testFamily/848a7772c59d474d8344f049156ef744, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:12,672 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe49dfd0c5ac08c96231aaa9c6b37364 in 76ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:12,697 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScannerAsync/fe49dfd0c5ac08c96231aaa9c6b37364/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:12,698 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:29:12,698 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fe49dfd0c5ac08c96231aaa9c6b37364: Waiting for close lock at 1733646552596Running coprocessor pre-close hooks at 1733646552596Disabling compacts and flushes for region at 1733646552596Disabling writes for close at 1733646552596Obtaining lock to block concurrent updates at 1733646552597 (+1 ms)Preparing flush snapshotting stores in fe49dfd0c5ac08c96231aaa9c6b37364 at 1733646552597Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646552597Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. at 1733646552599 (+2 ms)Flushing fe49dfd0c5ac08c96231aaa9c6b37364/testFamily: creating writer at 1733646552599Flushing fe49dfd0c5ac08c96231aaa9c6b37364/testFamily: appending metadata at 1733646552629 (+30 ms)Flushing fe49dfd0c5ac08c96231aaa9c6b37364/testFamily: closing flushed file at 1733646552629Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d292d5e: reopening flushed file at 1733646552656 (+27 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe49dfd0c5ac08c96231aaa9c6b37364 in 76ms, sequenceid=8, compaction requested=false at 1733646552672 (+16 ms)Writing region close event to WAL at 1733646552685 (+13 ms)Running coprocessor post-close hooks at 1733646552698 (+13 ms)Closed at 1733646552698 2024-12-08T08:29:12,698 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733646531734.fe49dfd0c5ac08c96231aaa9c6b37364. 2024-12-08T08:29:12,698 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4889d1457d6a210db6725bf12b2d7fd5, disabling compactions & flushes 2024-12-08T08:29:12,698 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:29:12,699 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:29:12,699 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. after waiting 0 ms 2024-12-08T08:29:12,699 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:29:12,699 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4889d1457d6a210db6725bf12b2d7fd5 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:12,702 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:29:12,720 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5/.tmp/testFamily/98966baebf8b4b7bb0bd8896a3781f1f is 61, key is row-0/testFamily:testQualifier/1733646506347/Put/seqid=0 2024-12-08T08:29:12,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741865_1041 (size=5049) 2024-12-08T08:29:12,902 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:29:13,102 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-08T08:29:13,102 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1325): Online Regions={4889d1457d6a210db6725bf12b2d7fd5=testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., 07499732554247c1fe88f68b036dfcc1=testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1., 3e6284cd8ab1289d786b0169bcbbc74d=testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.} 2024-12-08T08:29:13,102 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d, 4889d1457d6a210db6725bf12b2d7fd5 2024-12-08T08:29:13,129 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5/.tmp/testFamily/98966baebf8b4b7bb0bd8896a3781f1f 2024-12-08T08:29:13,142 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5/.tmp/testFamily/98966baebf8b4b7bb0bd8896a3781f1f as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5/testFamily/98966baebf8b4b7bb0bd8896a3781f1f 2024-12-08T08:29:13,157 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5/testFamily/98966baebf8b4b7bb0bd8896a3781f1f, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:13,158 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4889d1457d6a210db6725bf12b2d7fd5 in 459ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:13,173 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnOpenScanner/4889d1457d6a210db6725bf12b2d7fd5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:13,174 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:29:13,174 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4889d1457d6a210db6725bf12b2d7fd5: Waiting for close lock at 1733646552698Running coprocessor pre-close hooks at 1733646552698Disabling compacts and flushes for region at 1733646552698Disabling writes for close at 1733646552699 (+1 ms)Obtaining lock to block concurrent updates at 1733646552699Preparing flush snapshotting stores in 4889d1457d6a210db6725bf12b2d7fd5 at 1733646552699Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646552699Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. at 1733646552700 (+1 ms)Flushing 4889d1457d6a210db6725bf12b2d7fd5/testFamily: creating writer at 1733646552700Flushing 4889d1457d6a210db6725bf12b2d7fd5/testFamily: appending metadata at 1733646552720 (+20 ms)Flushing 4889d1457d6a210db6725bf12b2d7fd5/testFamily: closing flushed file at 1733646552720Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39e48c34: reopening flushed file at 1733646553140 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4889d1457d6a210db6725bf12b2d7fd5 in 459ms, sequenceid=8, compaction requested=false at 1733646553158 (+18 ms)Writing region close event to WAL at 1733646553161 (+3 ms)Running coprocessor post-close hooks at 1733646553174 (+13 ms)Closed at 1733646553174 2024-12-08T08:29:13,175 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733646505312.4889d1457d6a210db6725bf12b2d7fd5. 2024-12-08T08:29:13,175 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 07499732554247c1fe88f68b036dfcc1, disabling compactions & flushes 2024-12-08T08:29:13,175 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:29:13,175 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:29:13,175 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. after waiting 0 ms 2024-12-08T08:29:13,175 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:29:13,175 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 07499732554247c1fe88f68b036dfcc1 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:13,207 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1/.tmp/testFamily/821ba1aeab8f4092a4a29bf79546f969 is 61, key is row-0/testFamily:testQualifier/1733646485895/Put/seqid=0 2024-12-08T08:29:13,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741866_1042 (size=5049) 2024-12-08T08:29:13,303 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:29:13,503 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 07499732554247c1fe88f68b036dfcc1, 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:29:13,619 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1/.tmp/testFamily/821ba1aeab8f4092a4a29bf79546f969 2024-12-08T08:29:13,632 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1/.tmp/testFamily/821ba1aeab8f4092a4a29bf79546f969 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1/testFamily/821ba1aeab8f4092a4a29bf79546f969 2024-12-08T08:29:13,643 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1/testFamily/821ba1aeab8f4092a4a29bf79546f969, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:13,644 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 07499732554247c1fe88f68b036dfcc1 in 469ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:13,662 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testRetryOutOfOrderScannerNextExceptionAsync/07499732554247c1fe88f68b036dfcc1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:13,663 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:29:13,663 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 07499732554247c1fe88f68b036dfcc1: Waiting for close lock at 1733646553175Running coprocessor pre-close hooks at 1733646553175Disabling compacts and flushes for region at 1733646553175Disabling writes for close at 1733646553175Obtaining lock to block concurrent updates at 1733646553175Preparing flush snapshotting stores in 07499732554247c1fe88f68b036dfcc1 at 1733646553175Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646553176 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. at 1733646553177 (+1 ms)Flushing 07499732554247c1fe88f68b036dfcc1/testFamily: creating writer at 1733646553177Flushing 07499732554247c1fe88f68b036dfcc1/testFamily: appending metadata at 1733646553207 (+30 ms)Flushing 07499732554247c1fe88f68b036dfcc1/testFamily: closing flushed file at 1733646553207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d51950a: reopening flushed file at 1733646553630 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 07499732554247c1fe88f68b036dfcc1 in 469ms, sequenceid=8, compaction requested=false at 1733646553644 (+14 ms)Writing region close event to WAL at 1733646553645 (+1 ms)Running coprocessor post-close hooks at 1733646553663 (+18 ms)Closed at 1733646553663 2024-12-08T08:29:13,663 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733646482796.07499732554247c1fe88f68b036dfcc1. 2024-12-08T08:29:13,664 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3e6284cd8ab1289d786b0169bcbbc74d, disabling compactions & flushes 2024-12-08T08:29:13,664 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:29:13,664 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:29:13,664 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. after waiting 0 ms 2024-12-08T08:29:13,664 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:29:13,664 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3e6284cd8ab1289d786b0169bcbbc74d 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T08:29:13,690 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d/.tmp/testFamily/174de58bd57645828e049a8670976dc9 is 61, key is row-0/testFamily:testQualifier/1733646510425/Put/seqid=0 2024-12-08T08:29:13,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741867_1043 (size=5049) 2024-12-08T08:29:13,703 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:29:13,904 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:29:14,097 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d/.tmp/testFamily/174de58bd57645828e049a8670976dc9 2024-12-08T08:29:14,104 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T08:29:14,104 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1325): Online Regions={3e6284cd8ab1289d786b0169bcbbc74d=testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d.} 2024-12-08T08:29:14,104 DEBUG [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1351): Waiting on 3e6284cd8ab1289d786b0169bcbbc74d 2024-12-08T08:29:14,106 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d/.tmp/testFamily/174de58bd57645828e049a8670976dc9 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d/testFamily/174de58bd57645828e049a8670976dc9 2024-12-08T08:29:14,115 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d/testFamily/174de58bd57645828e049a8670976dc9, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T08:29:14,116 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3e6284cd8ab1289d786b0169bcbbc74d in 452ms, sequenceid=8, compaction requested=false 2024-12-08T08:29:14,122 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/data/default/testNormalScanTimeoutOnNextAsync/3e6284cd8ab1289d786b0169bcbbc74d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T08:29:14,123 INFO [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:29:14,123 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3e6284cd8ab1289d786b0169bcbbc74d: Waiting for close lock at 1733646553664Running coprocessor pre-close hooks at 1733646553664Disabling compacts and flushes for region at 1733646553664Disabling writes for close at 1733646553664Obtaining lock to block concurrent updates at 1733646553664Preparing flush snapshotting stores in 3e6284cd8ab1289d786b0169bcbbc74d at 1733646553664Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733646553664Flushing stores of testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. at 1733646553666 (+2 ms)Flushing 3e6284cd8ab1289d786b0169bcbbc74d/testFamily: creating writer at 1733646553666Flushing 3e6284cd8ab1289d786b0169bcbbc74d/testFamily: appending metadata at 1733646553689 (+23 ms)Flushing 3e6284cd8ab1289d786b0169bcbbc74d/testFamily: closing flushed file at 1733646553689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34728575: reopening flushed file at 1733646554105 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3e6284cd8ab1289d786b0169bcbbc74d in 452ms, sequenceid=8, compaction requested=false at 1733646554116 (+11 ms)Writing region close event to WAL at 1733646554117 (+1 ms)Running coprocessor post-close hooks at 1733646554122 (+5 ms)Closed at 1733646554122 2024-12-08T08:29:14,123 DEBUG [RS_CLOSE_REGION-regionserver/422e41b75614:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733646509397.3e6284cd8ab1289d786b0169bcbbc74d. 2024-12-08T08:29:14,305 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(976): stopping server 422e41b75614,32911,1733646448831; all regions closed. 2024-12-08T08:29:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741834_1010 (size=19479) 2024-12-08T08:29:14,317 DEBUG [RS:0;422e41b75614:32911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/oldWALs 2024-12-08T08:29:14,317 INFO [RS:0;422e41b75614:32911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 422e41b75614%2C32911%2C1733646448831.meta:.meta(num 1733646482316) 2024-12-08T08:29:14,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741833_1009 (size=20031) 2024-12-08T08:29:14,328 DEBUG [RS:0;422e41b75614:32911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/oldWALs 2024-12-08T08:29:14,328 INFO [RS:0;422e41b75614:32911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 422e41b75614%2C32911%2C1733646448831:(num 1733646481866) 2024-12-08T08:29:14,329 DEBUG [RS:0;422e41b75614:32911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T08:29:14,329 INFO [RS:0;422e41b75614:32911 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T08:29:14,329 INFO [RS:0;422e41b75614:32911 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T08:29:14,329 INFO [RS:0;422e41b75614:32911 {}] hbase.ChoreService(370): Chore service for: regionserver/422e41b75614:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T08:29:14,329 INFO [RS:0;422e41b75614:32911 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T08:29:14,329 INFO [regionserver/422e41b75614:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T08:29:14,330 INFO [regionserver/422e41b75614:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T08:29:14,331 INFO [RS:0;422e41b75614:32911 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:32911 2024-12-08T08:29:14,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/422e41b75614,32911,1733646448831 2024-12-08T08:29:14,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T08:29:14,399 INFO [RS:0;422e41b75614:32911 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T08:29:14,401 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [422e41b75614,32911,1733646448831] 2024-12-08T08:29:14,410 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/422e41b75614,32911,1733646448831 already deleted, retry=false 2024-12-08T08:29:14,410 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 422e41b75614,32911,1733646448831 expired; onlineServers=0 2024-12-08T08:29:14,410 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '422e41b75614,33175,1733646448021' ***** 2024-12-08T08:29:14,410 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T08:29:14,410 INFO [M:0;422e41b75614:33175 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T08:29:14,411 INFO [M:0;422e41b75614:33175 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T08:29:14,411 DEBUG [M:0;422e41b75614:33175 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T08:29:14,411 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T08:29:14,411 DEBUG [M:0;422e41b75614:33175 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T08:29:14,411 DEBUG [master/422e41b75614:0:becomeActiveMaster-HFileCleaner.small.0-1733646451687 {}] cleaner.HFileCleaner(306): Exit Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.small.0-1733646451687,5,FailOnTimeoutGroup] 2024-12-08T08:29:14,411 DEBUG [master/422e41b75614:0:becomeActiveMaster-HFileCleaner.large.0-1733646451686 {}] cleaner.HFileCleaner(306): Exit Thread[master/422e41b75614:0:becomeActiveMaster-HFileCleaner.large.0-1733646451686,5,FailOnTimeoutGroup] 2024-12-08T08:29:14,411 INFO [M:0;422e41b75614:33175 {}] hbase.ChoreService(370): Chore service for: master/422e41b75614:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T08:29:14,411 INFO [M:0;422e41b75614:33175 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T08:29:14,411 DEBUG [M:0;422e41b75614:33175 {}] master.HMaster(1795): Stopping service threads 2024-12-08T08:29:14,411 INFO [M:0;422e41b75614:33175 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T08:29:14,412 INFO [M:0;422e41b75614:33175 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T08:29:14,412 INFO [M:0;422e41b75614:33175 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T08:29:14,412 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T08:29:14,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T08:29:14,431 DEBUG [M:0;422e41b75614:33175 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-08T08:29:14,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T08:29:14,431 DEBUG [M:0;422e41b75614:33175 {}] master.ActiveMasterManager(353): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-08T08:29:14,433 INFO [M:0;422e41b75614:33175 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/.lastflushedseqids 2024-12-08T08:29:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741868_1044 (size=721) 2024-12-08T08:29:14,448 INFO [M:0;422e41b75614:33175 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T08:29:14,448 INFO [M:0;422e41b75614:33175 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T08:29:14,449 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T08:29:14,449 INFO [M:0;422e41b75614:33175 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:29:14,449 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:29:14,449 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T08:29:14,449 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:29:14,449 INFO [M:0;422e41b75614:33175 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.17 KB heapSize=191.21 KB 2024-12-08T08:29:14,476 DEBUG [M:0;422e41b75614:33175 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9d15d3c1365a4ce4bce4f1d29980f562 is 82, key is hbase:meta,,1/info:regioninfo/1733646482392/Put/seqid=0 2024-12-08T08:29:14,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741869_1045 (size=5672) 2024-12-08T08:29:14,486 INFO [M:0;422e41b75614:33175 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9d15d3c1365a4ce4bce4f1d29980f562 2024-12-08T08:29:14,519 DEBUG [M:0;422e41b75614:33175 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6ab77c4263f4ae0bbf9cc0ad31af180 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733646484058/Put/seqid=0 2024-12-08T08:29:14,521 INFO [RS:0;422e41b75614:32911 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T08:29:14,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:29:14,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32911-0x100048978dd0001, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:29:14,522 INFO [RS:0;422e41b75614:32911 {}] regionserver.HRegionServer(1031): Exiting; stopping=422e41b75614,32911,1733646448831; zookeeper connection closed. 2024-12-08T08:29:14,522 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6166305 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6166305 2024-12-08T08:29:14,523 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T08:29:14,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741870_1046 (size=14077) 2024-12-08T08:29:14,927 INFO [M:0;422e41b75614:33175 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.62 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6ab77c4263f4ae0bbf9cc0ad31af180 2024-12-08T08:29:14,943 INFO [M:0;422e41b75614:33175 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d6ab77c4263f4ae0bbf9cc0ad31af180 2024-12-08T08:29:14,972 DEBUG [M:0;422e41b75614:33175 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e269700a717d43f4985c481a009165e4 is 69, key is 422e41b75614,32911,1733646448831/rs:state/1733646451728/Put/seqid=0 2024-12-08T08:29:15,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741871_1047 (size=5156) 2024-12-08T08:29:15,408 INFO [M:0;422e41b75614:33175 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e269700a717d43f4985c481a009165e4 2024-12-08T08:29:15,419 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9d15d3c1365a4ce4bce4f1d29980f562 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9d15d3c1365a4ce4bce4f1d29980f562 2024-12-08T08:29:15,427 INFO [M:0;422e41b75614:33175 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9d15d3c1365a4ce4bce4f1d29980f562, entries=8, sequenceid=319, filesize=5.5 K 2024-12-08T08:29:15,428 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d6ab77c4263f4ae0bbf9cc0ad31af180 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d6ab77c4263f4ae0bbf9cc0ad31af180 2024-12-08T08:29:15,437 INFO [M:0;422e41b75614:33175 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d6ab77c4263f4ae0bbf9cc0ad31af180 2024-12-08T08:29:15,437 INFO [M:0;422e41b75614:33175 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d6ab77c4263f4ae0bbf9cc0ad31af180, entries=33, sequenceid=319, filesize=13.7 K 2024-12-08T08:29:15,438 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e269700a717d43f4985c481a009165e4 as hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e269700a717d43f4985c481a009165e4 2024-12-08T08:29:15,447 INFO [M:0;422e41b75614:33175 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33261/user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e269700a717d43f4985c481a009165e4, entries=1, sequenceid=319, filesize=5.0 K 2024-12-08T08:29:15,448 INFO [M:0;422e41b75614:33175 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.17 KB/167087, heapSize ~190.91 KB/195496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 999ms, sequenceid=319, compaction requested=false 2024-12-08T08:29:15,456 INFO [M:0;422e41b75614:33175 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T08:29:15,457 DEBUG [M:0;422e41b75614:33175 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733646554449Disabling compacts and flushes for region at 1733646554449Disabling writes for close at 1733646554449Obtaining lock to block concurrent updates at 1733646554449Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733646554449Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167087, getHeapSize=195736, getOffHeapSize=0, getCellsCount=366 at 1733646554449Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733646554450 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733646554450Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733646554474 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733646554475 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733646554495 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733646554517 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733646554518 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733646554943 (+425 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733646554971 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733646554971Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3290b0eb: reopening flushed file at 1733646555417 (+446 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fc6af59: reopening flushed file at 1733646555427 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542f48ec: reopening flushed file at 1733646555437 (+10 ms)Finished flush of dataSize ~163.17 KB/167087, heapSize ~190.91 KB/195496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 999ms, sequenceid=319, compaction requested=false at 1733646555448 (+11 ms)Writing region close event to WAL at 1733646555456 (+8 ms)Closed at 1733646555456 2024-12-08T08:29:15,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37723 is added to blk_1073741830_1006 (size=188429) 2024-12-08T08:29:15,461 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/619c8972-dd4c-c17e-de9f-1e55755dccd4/MasterData/WALs/422e41b75614,33175,1733646448021/422e41b75614%2C33175%2C1733646448021.1733646450972 not finished, retry = 0 2024-12-08T08:29:15,563 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T08:29:15,563 INFO [M:0;422e41b75614:33175 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T08:29:15,564 INFO [M:0;422e41b75614:33175 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33175 2024-12-08T08:29:15,564 INFO [M:0;422e41b75614:33175 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T08:29:15,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:29:15,710 INFO [M:0;422e41b75614:33175 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T08:29:15,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33175-0x100048978dd0000, quorum=127.0.0.1:60702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T08:29:15,745 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@729134c5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T08:29:15,748 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e0faa97{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T08:29:15,748 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T08:29:15,749 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@53038108{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T08:29:15,749 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c9b6fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/hadoop.log.dir/,STOPPED} 2024-12-08T08:29:15,754 WARN [BP-242960261-172.17.0.3-1733646444019 heartbeating to localhost/127.0.0.1:33261 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T08:29:15,754 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T08:29:15,754 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T08:29:15,754 WARN [BP-242960261-172.17.0.3-1733646444019 heartbeating to localhost/127.0.0.1:33261 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-242960261-172.17.0.3-1733646444019 (Datanode Uuid c92ae55d-6ed0-427f-801d-bf23bb444ec7) service to localhost/127.0.0.1:33261 2024-12-08T08:29:15,756 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/data/data1/current/BP-242960261-172.17.0.3-1733646444019 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T08:29:15,756 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/cluster_850b3e17-5e13-f101-d3a9-cefdc508f442/data/data2/current/BP-242960261-172.17.0.3-1733646444019 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T08:29:15,756 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T08:29:15,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bc299d4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T08:29:15,765 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e20720d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T08:29:15,766 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T08:29:15,766 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@734175dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T08:29:15,766 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6193f924{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/35f7d906-424e-5044-b88c-19b77f407df3/hadoop.log.dir/,STOPPED} 2024-12-08T08:29:15,784 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T08:29:15,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down